init: Prefect ETL sync project (RIIN/SDS → PG) with 12 sync flows
This commit is contained in:
yeuimu
@@ -0,0 +1,27 @@
|
||||
"""同步编排共享层:客户端/分页、映射、PG 写入。"""
|
||||
|
||||
from flows.shared import clients, db, mappers
|
||||
from flows.shared.clients import (
|
||||
create_riin_client,
|
||||
create_sds_client,
|
||||
fetch_goods_concurrent,
|
||||
fetch_riin_all_pages,
|
||||
fetch_sds_all_pages,
|
||||
floor_to_hour,
|
||||
format_date,
|
||||
get_channel_type_map,
|
||||
refresh_channel_type_map,
|
||||
resolve_courier_source,
|
||||
start_of_day,
|
||||
)
|
||||
from flows.shared.db import SyncDB
|
||||
from flows.shared.mappers import GoodsUpsertInput, OrderUpsertInput
|
||||
|
||||
__all__ = [
|
||||
"clients", "db", "mappers", "SyncDB",
|
||||
"OrderUpsertInput", "GoodsUpsertInput",
|
||||
"create_riin_client", "create_sds_client",
|
||||
"fetch_riin_all_pages", "fetch_sds_all_pages", "fetch_goods_concurrent",
|
||||
"get_channel_type_map", "refresh_channel_type_map", "resolve_courier_source",
|
||||
"format_date", "start_of_day", "floor_to_hour",
|
||||
]
|
||||
@@ -0,0 +1,229 @@
|
||||
"""同步共享:客户端工厂、顺序分页(含重试/延迟)、RIIN 商品拉取、SDS 物流渠道字典、日期辅助。
|
||||
|
||||
行为对齐 inkreach-bi-auth-mng 的 sync/tasks/clients.ts:
|
||||
- 顺序分页(页间 500ms),非并发,对外部 API 友好。
|
||||
- fetch_with_retry:仅对 5xx / 429 / 超时重试,最多 3 次,退避 3s*次数。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Callable, TypeVar
|
||||
|
||||
from prefect.blocks.system import Secret
|
||||
|
||||
from api import RiinDisClient, SdsMngClient
|
||||
from api.riin.models import OrderPageRequest as RiinReq
|
||||
from api.sds.models import LogisticsPageRequest, OrderPageRequest as SdsReq
|
||||
from flows.shared.mappers import map_sds_channel_type
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
GOODS_CONCURRENCY = 20
|
||||
PAGE_DELAY_MS = 500
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAY_MS = 3000
|
||||
TZ_CN = timezone(timedelta(hours=8))
|
||||
|
||||
_RIIN_FINISHED = ["已退款", "已关闭", "已发货"]
|
||||
_SDS_FINISHED = ["已完成", "已取消"]
|
||||
|
||||
|
||||
# ---------- 客户端工厂 ----------
|
||||
|
||||
def create_riin_client() -> RiinDisClient:
|
||||
cred = _load_secret("riin-credentials")
|
||||
return RiinDisClient(cred["username"], cred["password"])
|
||||
|
||||
|
||||
def create_sds_client() -> SdsMngClient:
|
||||
cred = _load_secret("sds-credentials")
|
||||
return SdsMngClient(cred["username"], cred["password"])
|
||||
|
||||
|
||||
def _load_secret(name: str) -> dict[str, str]:
|
||||
value = Secret.load(name).get()
|
||||
if isinstance(value, str):
|
||||
value = json.loads(value)
|
||||
return value
|
||||
|
||||
|
||||
# ---------- 日期辅助 ----------
|
||||
|
||||
def format_date(d: datetime) -> str:
|
||||
return d.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def start_of_day(d: datetime) -> datetime:
|
||||
return d.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
def floor_to_hour(d: datetime) -> datetime:
|
||||
return d.replace(minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
# ---------- 重试 ----------
|
||||
|
||||
def fetch_with_retry(fn: Callable[[], T], label: str) -> T:
|
||||
last_err: Exception | None = None
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
return fn()
|
||||
except Exception as err: # noqa: BLE001
|
||||
last_err = err
|
||||
status = _status_of(err)
|
||||
msg = str(err)
|
||||
is_timeout = "timeout" in msg.lower() or "timed out" in msg.lower()
|
||||
retryable = status == 429 or (status is not None and status >= 500) or is_timeout
|
||||
if attempt < MAX_RETRIES and retryable:
|
||||
print(f"[sync] {label} 第 {attempt}/{MAX_RETRIES} 次重试 ({status}: {msg[:120]})")
|
||||
time.sleep(RETRY_DELAY_MS * attempt / 1000)
|
||||
continue
|
||||
raise
|
||||
assert last_err is not None
|
||||
raise last_err
|
||||
|
||||
|
||||
def _status_of(err: Exception) -> int | None:
|
||||
for attr in ("status_code", "statusCode", "status"):
|
||||
v = getattr(err, attr, None)
|
||||
if isinstance(v, int):
|
||||
return v
|
||||
resp = getattr(err, "response", None)
|
||||
return getattr(resp, "status_code", None) if resp is not None else None
|
||||
|
||||
|
||||
# ---------- 分页 ----------
|
||||
|
||||
def fetch_riin_all_pages(client: RiinDisClient, params: dict[str, Any], max_pages: int = 1000) -> list[Any]:
|
||||
page_size = params.get("page_size", 200)
|
||||
base = {**params, "page_size": page_size}
|
||||
|
||||
def fetch(idx: int):
|
||||
return fetch_with_retry(lambda: client.get_orders(RiinReq(**{**base, "page_index": idx})), f"RIIN page {idx}")
|
||||
|
||||
first = fetch(1)
|
||||
total = first.data.total
|
||||
records = list(first.data.records)
|
||||
total_pages = min(-(-total // page_size), max_pages)
|
||||
for page in range(2, total_pages + 1):
|
||||
resp = fetch(page)
|
||||
records.extend(resp.data.records)
|
||||
time.sleep(PAGE_DELAY_MS / 1000)
|
||||
if total_pages == max_pages:
|
||||
print(f"[RIIN sync] 达到最大页数限制 {max_pages},截断到 {len(records)} 条")
|
||||
return records
|
||||
|
||||
|
||||
def fetch_sds_all_pages(client: SdsMngClient, params: dict[str, Any]) -> list[Any]:
|
||||
page_size = params.get("size", 500)
|
||||
base = {**params, "size": page_size}
|
||||
|
||||
def fetch(idx: int):
|
||||
return fetch_with_retry(lambda: client.get_orders(SdsReq(**{**base, "page": idx})), f"SDS page {idx}")
|
||||
|
||||
first = fetch(1)
|
||||
records = list(first.records)
|
||||
total_pages = -(-first.total_count // page_size)
|
||||
for page in range(2, total_pages + 1):
|
||||
resp = fetch(page)
|
||||
records.extend(resp.records)
|
||||
time.sleep(PAGE_DELAY_MS / 1000)
|
||||
return records
|
||||
|
||||
|
||||
# ---------- RIIN 商品 ----------
|
||||
|
||||
def fetch_goods_for_order(client: RiinDisClient, order_id: str) -> list[Any]:
|
||||
try:
|
||||
resp = client.get_order_goods(order_id)
|
||||
return list(resp.data)
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[RIIN sync] 订单 {order_id} 商品拉取失败: {err}")
|
||||
return []
|
||||
|
||||
|
||||
def fetch_goods_concurrent(client: RiinDisClient, order_ids: list[str]) -> dict[str, list[Any]]:
|
||||
"""以 GOODS_CONCURRENCY 并发拉取多订单商品,返回 {order_id: [goods...]}。"""
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
results: dict[str, list[Any]] = {}
|
||||
if not order_ids:
|
||||
return results
|
||||
|
||||
def make(oid: str) -> Callable[[], None]:
|
||||
def run() -> None:
|
||||
results[oid] = fetch_goods_for_order(client, oid)
|
||||
return run
|
||||
|
||||
with ThreadPoolExecutor(max_workers=min(GOODS_CONCURRENCY, len(order_ids))) as pool:
|
||||
list(pool.map(lambda fn: fn(), [make(oid) for oid in order_ids]))
|
||||
return results
|
||||
|
||||
|
||||
# ---------- SDS 物流渠道字典 ----------
|
||||
|
||||
_channel_cache: dict[str, Any] = {}
|
||||
_CHANNEL_TTL = 24 * 3600
|
||||
_CHANNEL_KEY = "all"
|
||||
|
||||
|
||||
def get_channel_type_map(client: SdsMngClient) -> dict[int, str | None]:
|
||||
cached = _channel_cache.get(_CHANNEL_KEY)
|
||||
if cached and cached["expire_at"] > time.time():
|
||||
return cached["data"]
|
||||
mapping: dict[int, str | None] = {}
|
||||
try:
|
||||
areas = client.get_logistics_areas()
|
||||
area_ids = [a.id for a in areas.items if a.id is not None]
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[SDS sync] 物流区域拉取失败: {err}")
|
||||
area_ids = []
|
||||
for area_id in area_ids:
|
||||
try:
|
||||
resp = client.get_logistics_page(LogisticsPageRequest(issuing_bay_area_id=area_id, all=True, size=200))
|
||||
for item in resp.items:
|
||||
if item.id is None:
|
||||
continue
|
||||
raw = getattr(item, "channelType", None) or getattr(item, "channel_type", None)
|
||||
label = map_sds_channel_type(raw)
|
||||
if label is None:
|
||||
continue
|
||||
mapping[int(item.id)] = label
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[SDS sync] 区域 {area_id} 物流字典拉取失败: {err}")
|
||||
_channel_cache[_CHANNEL_KEY] = {"data": mapping, "expire_at": time.time() + _CHANNEL_TTL}
|
||||
return mapping
|
||||
|
||||
|
||||
def refresh_channel_type_map(client: SdsMngClient) -> dict[int, str | None]:
|
||||
_channel_cache.pop(_CHANNEL_KEY, None)
|
||||
return get_channel_type_map(client)
|
||||
|
||||
|
||||
def resolve_courier_source(
|
||||
raw_logistics_id: Any,
|
||||
mapping: dict[int, str | None],
|
||||
refresh: Callable[[], dict[int, str | None]],
|
||||
refreshed_ids: set[int],
|
||||
) -> str | None:
|
||||
if raw_logistics_id is None:
|
||||
return None
|
||||
try:
|
||||
lid = int(raw_logistics_id)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if lid in mapping:
|
||||
return mapping[lid]
|
||||
if lid in refreshed_ids:
|
||||
return None
|
||||
refreshed_ids.add(lid)
|
||||
try:
|
||||
new_map = refresh()
|
||||
mapping.update(new_map)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return mapping.get(lid)
|
||||
@@ -0,0 +1,365 @@
|
||||
"""PG 同步写入层:Orders / Goods / Merchant 的 upsert + 同步所需的查询。
|
||||
|
||||
忠实移植自 inkreach-bi-auth-mng 的 sync/lib/db.service.ts。
|
||||
- Orders 主键:("orderNo", "createdAt")
|
||||
- Goods 主键:("orderNo", "productionNo")
|
||||
关键:写入前用库中已有 createdAt 覆盖,避免同一订单因 createdAt 不同产生重复主键行。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from psycopg2.extras import execute_values
|
||||
|
||||
from flows.shared.mappers import GoodsUpsertInput, OrderUpsertInput
|
||||
|
||||
ORDER_CHUNK = 1000
|
||||
GOODS_CHUNK = 2000
|
||||
LOOKUP_BATCH = 10000
|
||||
|
||||
|
||||
class SyncDB:
|
||||
def __init__(self, block_name: str = "pg-inkreach-dmp") -> None:
|
||||
from prefect_sqlalchemy import SqlAlchemyConnector
|
||||
|
||||
self._engine = SqlAlchemyConnector.load(block_name).get_engine()
|
||||
|
||||
def _conn(self):
|
||||
return self._engine.raw_connection()
|
||||
|
||||
# ---------- 查询辅助 ----------
|
||||
|
||||
def get_latest_ordered_at(self, system_name: str) -> datetime | None:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT MAX(COALESCE("orderedAt","createdAt")) FROM "Orders" WHERE "systemName"=%s',
|
||||
(system_name,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return row[0] if row and row[0] else None
|
||||
|
||||
def get_pending_orders(self, system_name: str, exclude_statuses: list[str]) -> list[str]:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s '
|
||||
'AND "systemStatus" IS DISTINCT FROM ALL %s',
|
||||
(system_name, exclude_statuses),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
def get_order_count_by_status(self, system_name: str, start: datetime, end: datetime) -> dict[str, int]:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT "systemStatus", COUNT(*) FROM "Orders" WHERE "systemName"=%s '
|
||||
'AND "orderedAt">=%s AND "orderedAt"<%s GROUP BY "systemStatus"',
|
||||
(system_name, start, end),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return {r[0]: int(r[1]) for r in rows}
|
||||
|
||||
def get_local_order_nos(self, system_name: str, start: datetime, end: datetime) -> list[str]:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s '
|
||||
'AND "orderedAt">=%s AND "orderedAt"<%s',
|
||||
(system_name, start, end),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
def get_local_goods_count(self, system_name: str, start: datetime, end: datetime) -> int:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT COUNT(*) FROM "Goods" g JOIN "Orders" o '
|
||||
'ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" '
|
||||
'WHERE o."systemName"=%s AND o."orderedAt">=%s AND o."orderedAt"<%s',
|
||||
(system_name, start, end),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
def get_orders_with_100_prefix(self, system_name: str, statuses: list[str]) -> list[tuple[str, datetime]]:
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT DISTINCT o."orderNo", o."createdAt" FROM "Orders" o '
|
||||
'JOIN "Goods" g ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" '
|
||||
'WHERE o."systemName"=%s AND o."systemStatus"=ANY(%s) AND g."productionNo" LIKE \'100%\'',
|
||||
(system_name, statuses),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
return [(r[0], r[1]) for r in rows]
|
||||
|
||||
# ---------- 主 upsert ----------
|
||||
|
||||
def sync_orders_and_goods(
|
||||
self,
|
||||
orders: list[OrderUpsertInput],
|
||||
goods: list[GoodsUpsertInput],
|
||||
delete_order_nos: list[str] | None = None,
|
||||
) -> dict[str, int]:
|
||||
result = {
|
||||
"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0,
|
||||
"goodsUpdated": 0, "goodsRemoved": 0, "ordersDeleted": 0,
|
||||
}
|
||||
if not orders and not goods and not delete_order_nos:
|
||||
return result
|
||||
|
||||
conn = self._conn()
|
||||
try:
|
||||
system_name = orders[0].systemName if orders else None
|
||||
all_order_nos = list({o.orderNo for o in orders} | {g.orderNo for g in goods})
|
||||
|
||||
# 1. 解析已有 createdAt,覆盖到 orders(保证幂等 upsert)
|
||||
existing_created: dict[str, datetime] = {}
|
||||
for i in range(0, len(all_order_nos), LOOKUP_BATCH):
|
||||
chunk = all_order_nos[i : i + LOOKUP_BATCH]
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)'
|
||||
+ (f' AND "systemName"=%s' if system_name else ""),
|
||||
(chunk, system_name) if system_name else (chunk,),
|
||||
)
|
||||
for r in cur.fetchall():
|
||||
existing_created[r[0]] = r[1]
|
||||
for o in orders:
|
||||
if o.orderNo in existing_created:
|
||||
o.createdAt = existing_created[o.orderNo]
|
||||
|
||||
# 2. 去重 orders / goods
|
||||
seen_o: set[str] = set()
|
||||
orders_dedup: list[OrderUpsertInput] = []
|
||||
for o in orders:
|
||||
key = f'{o.orderNo}|{o.createdAt.isoformat()}'
|
||||
if key in seen_o:
|
||||
continue
|
||||
seen_o.add(key)
|
||||
orders_dedup.append(o)
|
||||
|
||||
resolved_created: dict[str, datetime] = {}
|
||||
for o in orders_dedup:
|
||||
resolved_created.setdefault(o.orderNo, o.createdAt)
|
||||
|
||||
seen_g: set[str] = set()
|
||||
goods_dedup: list[GoodsUpsertInput] = []
|
||||
for g in goods:
|
||||
if g.orderNo in resolved_created:
|
||||
g.createdAt = resolved_created[g.orderNo]
|
||||
key = f'{g.orderNo}|{g.productionNo}'
|
||||
if key in seen_g:
|
||||
continue
|
||||
seen_g.add(key)
|
||||
goods_dedup.append(g)
|
||||
|
||||
goods_by_order: dict[str, list[GoodsUpsertInput]] = {}
|
||||
for g in goods_dedup:
|
||||
goods_by_order.setdefault(g.orderNo, []).append(g)
|
||||
|
||||
# 3. 分批 upsert orders + 关联 goods
|
||||
for i in range(0, len(orders_dedup), ORDER_CHUNK):
|
||||
chunk = orders_dedup[i : i + ORDER_CHUNK]
|
||||
related: list[GoodsUpsertInput] = []
|
||||
for o in chunk:
|
||||
related.extend(goods_by_order.get(o.orderNo, []))
|
||||
stats = self._upsert_order_batch(conn, chunk, related, system_name)
|
||||
for k, v in stats.items():
|
||||
result[k] += v
|
||||
|
||||
# 4. 剩余 goods(其 order 不在本次 orders 列表)
|
||||
orders_set = {o.orderNo for o in orders_dedup}
|
||||
remaining = [g for g in goods_dedup if g.orderNo not in orders_set]
|
||||
if remaining:
|
||||
c, u = self._upsert_goods_only(conn, remaining, system_name)
|
||||
result["goodsCreated"] += c
|
||||
result["goodsUpdated"] += u
|
||||
|
||||
# 5. 删除订单
|
||||
if delete_order_nos:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'DELETE FROM "Orders" WHERE "orderNo"=ANY(%s)'
|
||||
+ (f' AND "systemName"=%s' if system_name else ""),
|
||||
(delete_order_nos, system_name) if system_name else (delete_order_nos,),
|
||||
)
|
||||
result["ordersDeleted"] = cur.rowcount
|
||||
|
||||
# 6. 商家占位
|
||||
self._ensure_merchants(conn, orders_dedup)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
return result
|
||||
|
||||
def _upsert_order_batch(
|
||||
self, conn, orders: list[OrderUpsertInput], goods: list[GoodsUpsertInput], system_name: str | None
|
||||
) -> dict[str, int]:
|
||||
stats = {"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0, "goodsUpdated": 0, "goodsRemoved": 0}
|
||||
with conn.cursor() as cur:
|
||||
# 统计已存在 → created/updated
|
||||
pairs = [(o.orderNo, o.createdAt) for o in orders]
|
||||
cur.execute(
|
||||
"SELECT t.order_no, t.created_at FROM unnest(%s::text[], %s::timestamp[]) AS t(order_no,created_at) "
|
||||
"JOIN \"Orders\" o ON o.\"orderNo\"=t.order_no AND o.\"createdAt\"=t.created_at",
|
||||
_unnest_pairs(pairs),
|
||||
)
|
||||
existing_set = {(r[0], r[1]) for r in cur.fetchall()}
|
||||
updated = sum(1 for o in orders if (o.orderNo, o.createdAt) in existing_set)
|
||||
stats["ordersUpdated"] = updated
|
||||
stats["ordersCreated"] = len(orders) - updated
|
||||
|
||||
execute_values(
|
||||
cur,
|
||||
'''INSERT INTO "Orders" (
|
||||
"orderNo","outOrderNo","courierSource","courierName","courierNo","merchantId",
|
||||
"countryRegion","factoryName","systemName","systemStatus","platformName","platformStatus",
|
||||
"createdAt","orderedAt","confirmedAt","completedAt","shipDeadline"
|
||||
) VALUES %s
|
||||
ON CONFLICT ("orderNo","createdAt") DO UPDATE SET
|
||||
"outOrderNo"=EXCLUDED."outOrderNo","courierSource"=EXCLUDED."courierSource",
|
||||
"courierName"=EXCLUDED."courierName","courierNo"=EXCLUDED."courierNo",
|
||||
"merchantId"=EXCLUDED."merchantId","countryRegion"=EXCLUDED."countryRegion",
|
||||
"factoryName"=EXCLUDED."factoryName","systemName"=EXCLUDED."systemName",
|
||||
"systemStatus"=EXCLUDED."systemStatus","platformName"=EXCLUDED."platformName",
|
||||
"platformStatus"=EXCLUDED."platformStatus","orderedAt"=EXCLUDED."orderedAt",
|
||||
"confirmedAt"=EXCLUDED."confirmedAt","completedAt"=EXCLUDED."completedAt",
|
||||
"shipDeadline"=EXCLUDED."shipDeadline"''',
|
||||
[(
|
||||
o.orderNo, o.outOrderNo, o.courierSource, o.courierName, o.courierNo, o.merchantId,
|
||||
o.countryRegion, o.factoryName, o.systemName, o.systemStatus, o.platformName, o.platformStatus,
|
||||
o.createdAt, o.orderedAt, o.confirmedAt, o.completedAt, o.shipDeadline,
|
||||
) for o in orders],
|
||||
page_size=ORDER_CHUNK,
|
||||
)
|
||||
|
||||
# 关联 goods:先删本次不再出现的,再 upsert
|
||||
if goods:
|
||||
order_nos = list({g.orderNo for g in goods})
|
||||
prod_nos = [g.productionNo for g in goods]
|
||||
cur.execute(
|
||||
'''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s)
|
||||
AND ("orderNo","createdAt") IN (
|
||||
SELECT "orderNo","createdAt" FROM "Orders"
|
||||
WHERE "orderNo"=ANY(%s)''' + (f' AND "systemName"=%s' if system_name else "") + ')',
|
||||
(order_nos, prod_nos, order_nos, system_name) if system_name else (order_nos, prod_nos, order_nos),
|
||||
)
|
||||
stats["goodsRemoved"] += cur.rowcount
|
||||
|
||||
g_pairs = [(g.orderNo, g.productionNo) for g in goods]
|
||||
cur.execute(
|
||||
"SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) "
|
||||
"JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no",
|
||||
_unnest_pairs(g_pairs),
|
||||
)
|
||||
g_existing = {(r[0], r[1]) for r in cur.fetchall()}
|
||||
g_updated = sum(1 for g in goods if (g.orderNo, g.productionNo) in g_existing)
|
||||
stats["goodsUpdated"] = g_updated
|
||||
stats["goodsCreated"] = len(goods) - g_updated
|
||||
|
||||
execute_values(
|
||||
cur,
|
||||
'''INSERT INTO "Goods" (
|
||||
"orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor",
|
||||
"itemSize","itemStyleNo","quantity","status"
|
||||
) VALUES %s
|
||||
ON CONFLICT ("orderNo","productionNo") DO UPDATE SET
|
||||
"createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo",
|
||||
"itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor",
|
||||
"itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo",
|
||||
"quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''',
|
||||
[(
|
||||
g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor,
|
||||
g.itemSize, g.itemStyleNo, g.quantity, g.status,
|
||||
) for g in goods],
|
||||
page_size=GOODS_CHUNK,
|
||||
)
|
||||
return stats
|
||||
|
||||
def _upsert_goods_only(
|
||||
self, conn, goods: list[GoodsUpsertInput], system_name: str | None
|
||||
) -> tuple[int, int]:
|
||||
# 用库中实际 createdAt 覆盖
|
||||
order_nos = list({g.orderNo for g in goods})
|
||||
actual: dict[str, datetime] = {}
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)'
|
||||
+ (f' AND "systemName"=%s' if system_name else ""),
|
||||
(order_nos, system_name) if system_name else (order_nos,),
|
||||
)
|
||||
for r in cur.fetchall():
|
||||
actual[r[0]] = r[1]
|
||||
for g in goods:
|
||||
if g.orderNo in actual:
|
||||
g.createdAt = actual[g.orderNo]
|
||||
|
||||
created = updated = 0
|
||||
for i in range(0, len(goods), GOODS_CHUNK):
|
||||
chunk = goods[i : i + GOODS_CHUNK]
|
||||
with conn.cursor() as cur:
|
||||
c_order = list({g.orderNo for g in chunk})
|
||||
c_prod = [g.productionNo for g in chunk]
|
||||
cur.execute(
|
||||
'''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s)'''
|
||||
+ (f''' AND ("orderNo","createdAt") IN (SELECT "orderNo","createdAt" FROM "Orders"
|
||||
WHERE "orderNo"=ANY(%s) AND "systemName"=%s)''' if system_name else ""),
|
||||
(c_order, c_prod, c_order, system_name) if system_name else (c_order, c_prod),
|
||||
)
|
||||
g_pairs = [(g.orderNo, g.productionNo) for g in chunk]
|
||||
cur.execute(
|
||||
"SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) "
|
||||
"JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no",
|
||||
_unnest_pairs(g_pairs),
|
||||
)
|
||||
g_existing = {(r[0], r[1]) for r in cur.fetchall()}
|
||||
u = sum(1 for g in chunk if (g.orderNo, g.productionNo) in g_existing)
|
||||
updated += u
|
||||
created += len(chunk) - u
|
||||
execute_values(
|
||||
cur,
|
||||
'''INSERT INTO "Goods" (
|
||||
"orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor",
|
||||
"itemSize","itemStyleNo","quantity","status"
|
||||
) VALUES %s
|
||||
ON CONFLICT ("orderNo","productionNo") DO UPDATE SET
|
||||
"createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo",
|
||||
"itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor",
|
||||
"itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo",
|
||||
"quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''',
|
||||
[(
|
||||
g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor,
|
||||
g.itemSize, g.itemStyleNo, g.quantity, g.status,
|
||||
) for g in chunk],
|
||||
page_size=GOODS_CHUNK,
|
||||
)
|
||||
return created, updated
|
||||
|
||||
def _ensure_merchants(self, conn, orders: list[OrderUpsertInput]) -> None:
|
||||
ids = list({o.merchantId for o in orders if o.merchantId and o.merchantId.strip()})
|
||||
if not ids:
|
||||
return
|
||||
with conn.cursor() as cur:
|
||||
execute_values(
|
||||
cur,
|
||||
'''INSERT INTO "Merchant" ("merchantId","merchantGroup","responsibleName","responsibleId","tags")
|
||||
VALUES %s ON CONFLICT ("merchantId") DO NOTHING''',
|
||||
[(mid, "", None, None, "") for mid in ids],
|
||||
)
|
||||
|
||||
|
||||
def _unnest_pairs(pairs: list[tuple]) -> tuple[list, list]:
|
||||
"""构造 psycopg2 unnest 双数组参数:返回 (col_a_list, col_b_list)。
|
||||
|
||||
配合 SQL:unnest(%s::text[], %s::timestamp[]) 或 unnest(%s::text[], %s::text[])。
|
||||
"""
|
||||
return [p[0] for p in pairs], [p[1] for p in pairs]
|
||||
@@ -0,0 +1,317 @@
|
||||
"""数据映射:RIIN / SDS 原始订单 → 统一 Order/Goods 写入格式。
|
||||
|
||||
忠实移植自 inkreach-bi-auth-mng 的 sync/mappers/{riin,sds,normalize-country}.ts。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
TZ_CN = timezone(timedelta(hours=8))
|
||||
|
||||
# ---------- RIIN 状态 / 平台 / 快递来源映射 ----------
|
||||
|
||||
RIIN_SYSTEM_STATUS_MAP: dict[int, str] = {
|
||||
-4: "待推送", -3: "工厂反审", -5: "待审核", -6: "分配失败",
|
||||
1: "待推送", 3: "已反审", 4: "生产中", 5: "生产中",
|
||||
12: "已发货", 13: "已关闭", 14: "已取消", 15: "已关闭",
|
||||
}
|
||||
|
||||
RIIN_PLATFORM_TYPE_MAP: dict[int, str] = {
|
||||
2: "淘宝", 3: "拼多多", 6: "Shopee", 7: "LAZADA", 8: "1688",
|
||||
11: "TIKTOK", 13: "TEMU", 15: "自建站(海外)", 16: "SHOPIFY",
|
||||
17: "SHEIN", 18: "自建站(国内)", 19: "WALMART", 21: "AMAZON",
|
||||
26: "OZON", 27: "Wildberries", 30: "ETSY", 31: "MERCADO",
|
||||
}
|
||||
|
||||
RIIN_EXPRESS_SOURCE_MAP: dict[int, str] = {
|
||||
1: "工厂发货", 2: "平台面单", 3: "自有面单", 4: "分销面单", 5: "电商面单",
|
||||
}
|
||||
|
||||
RIIN_GOODS_STATUS_MAP: dict[str, str] = {
|
||||
"NOT_SHIPPED": "待发货", "SHIPPED": "已发货", "CANCEL": "已取消",
|
||||
"CANCELLED": "已取消", "REFUNDING": "退款中",
|
||||
}
|
||||
|
||||
RIIN_PLATFORM_STATUS_MAP: dict[str, str] = {
|
||||
"SHIPPED": "已发货", "NOT_SHIPPED": "待发货", "CANCEL": "已取消",
|
||||
"PAUSE": "暂停", "COMPLETE": "已完成",
|
||||
}
|
||||
|
||||
# ---------- SDS 状态 / 物流渠道映射 ----------
|
||||
|
||||
SDS_SYSTEM_STATUS_MAP: dict[int, str] = {
|
||||
1: "待付款", 2: "待确认", 3: "备货中", 4: "已完成", 5: "搁置中",
|
||||
7: "待设计", 9: "部分发货", 98: "已取消",
|
||||
}
|
||||
SDS_GOODS_STATUS_MAP: dict[int, str] = {**SDS_SYSTEM_STATUS_MAP, 99: "已删除"}
|
||||
|
||||
SDS_CHANNEL_TYPE_LABEL_MAP: dict[str, str] = {
|
||||
"online": "线上物流", "offline": "普通物流", "manual": "手动物流", "self_pickup": "自提",
|
||||
}
|
||||
|
||||
|
||||
# ---------- 日期解析 ----------
|
||||
|
||||
def safe_date(value: Any, context: str = "") -> datetime:
|
||||
"""解析日期字符串(无时区视为 +08:00),返回 tz-aware datetime。"""
|
||||
if not value or not isinstance(value, str):
|
||||
raise ValueError(f"Invalid date for {context}: {value!r}")
|
||||
s = value.strip()
|
||||
if "+" not in s and "T" not in s and not s.endswith("Z"):
|
||||
s = f"{s}T00:00:00+08:00" if len(s) == 10 else f"{s}+08:00"
|
||||
try:
|
||||
dt = datetime.fromisoformat(s)
|
||||
except ValueError:
|
||||
raise ValueError(f"Invalid date for {context}: {value!r}")
|
||||
return dt if dt.tzinfo else dt.replace(tzinfo=TZ_CN)
|
||||
|
||||
|
||||
def _from_epoch_ms(value: Any) -> datetime | None:
|
||||
try:
|
||||
n = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return datetime.fromtimestamp(n / 1000, tz=TZ_CN) if n > 0 else None
|
||||
|
||||
|
||||
# ---------- 国家/地区归一 ----------
|
||||
|
||||
_US_ALIASES = {"US", "UNITED STATES", "USA", "ESTADOS UNIDOS", "美国加州发货仓"}
|
||||
_MX_ALIASES = {"MX", "MEXICO", "MÉXICO", "MEX", "ME"}
|
||||
_CA_ALIASES = {"加拿大工厂", "CA"}
|
||||
_PL_ALIASES = {"欧州波兰发货仓", "欧洲波兰法兰克福发货仓"}
|
||||
_WAREHOUSE_MAP = {
|
||||
"印美达澳洲发货仓": "澳洲", "欧洲德国发货仓": "德国", "欧洲匈牙利发货仓": "匈牙利",
|
||||
"欧洲西班牙发货仓": "西班牙", "欧洲意大利发货仓": "意大利",
|
||||
}
|
||||
_DIRECT_MAP = {"Saudi Arabia": "沙特", "Japan": "日本"}
|
||||
_EXCLUDED_COUNTRIES = {
|
||||
"Croatia", "Czech Republic", "丹麦", "保加利亚", "克罗地亚", "卢森堡", "奥地利", "希腊",
|
||||
"拉脱维亚", "捷克", "斯洛伐克", "斯洛文尼亚", "比利时", "法国", "爱尔兰", "爱沙尼亚",
|
||||
"瑞典", "瑞士", "立陶宛", "罗马尼亚", "芬兰", "荷兰", "Denmark", "Austria", "Greece",
|
||||
"Latvia", "Slovakia", "Slovenia", "Belgium", "France", "Ireland", "Estonia", "Sweden",
|
||||
"Switzerland", "Lithuania", "Romania", "Finland", "Netherlands", "Luxembourg", "Bulgaria",
|
||||
}
|
||||
|
||||
|
||||
def normalize_country_region(raw: str | None, factory_name: str | None = None) -> str | None:
|
||||
if factory_name == "广州印美达":
|
||||
return "中国"
|
||||
if not raw:
|
||||
return None
|
||||
trimmed = raw.strip()
|
||||
if trimmed in _EXCLUDED_COUNTRIES:
|
||||
return None
|
||||
upper = trimmed.upper()
|
||||
if upper in _US_ALIASES:
|
||||
return "美国"
|
||||
if upper in _MX_ALIASES:
|
||||
return "墨西哥"
|
||||
if upper in _CA_ALIASES:
|
||||
return "加拿大"
|
||||
if trimmed in _PL_ALIASES:
|
||||
return "波兰"
|
||||
if trimmed in _WAREHOUSE_MAP:
|
||||
return _WAREHOUSE_MAP[trimmed]
|
||||
if raw in _DIRECT_MAP:
|
||||
return _DIRECT_MAP[raw]
|
||||
return raw
|
||||
|
||||
|
||||
# ---------- 统一写入模型 ----------
|
||||
|
||||
class OrderUpsertInput:
|
||||
__slots__ = (
|
||||
"orderNo", "outOrderNo", "courierSource", "courierName", "courierNo",
|
||||
"merchantId", "countryRegion", "factoryName", "systemName", "systemStatus",
|
||||
"platformName", "platformStatus", "createdAt", "orderedAt", "confirmedAt",
|
||||
"completedAt", "shipDeadline",
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
for k in self.__slots__:
|
||||
setattr(self, k, kwargs.get(k))
|
||||
|
||||
|
||||
class GoodsUpsertInput:
|
||||
__slots__ = (
|
||||
"orderNo", "productionNo", "createdAt", "outOrderNo", "itemName",
|
||||
"itemColor", "itemSize", "itemStyleNo", "quantity", "status",
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
for k in self.__slots__:
|
||||
setattr(self, k, kwargs.get(k))
|
||||
|
||||
|
||||
# ---------- RIIN 映射 ----------
|
||||
|
||||
def map_riin_system_status(order_status: int) -> str | None:
|
||||
return RIIN_SYSTEM_STATUS_MAP.get(order_status)
|
||||
|
||||
|
||||
def map_riin_order(raw: Any) -> OrderUpsertInput:
|
||||
platform_oid = raw.platform_oid
|
||||
ordered_at = _parse(raw.push_time, f"RIIN order {platform_oid} pushTime")
|
||||
confirmed_at = _parse(raw.distributor_push_time, f"RIIN order {platform_oid} distributorPushTime")
|
||||
system_status = map_riin_system_status(raw.order_status) if raw.order_status is not None else None
|
||||
is_terminal = system_status in ("已发货", "已关闭", "已取消")
|
||||
completed_at: datetime | None = None
|
||||
if is_terminal:
|
||||
completed_at = _parse(raw.shipping_time, "") or _parse(raw.finish_time, "")
|
||||
ship_deadline = _parse(raw.except_last_shipping_time, "")
|
||||
courier_source = RIIN_EXPRESS_SOURCE_MAP.get(raw.express_source) if raw.express_source is not None else None
|
||||
factory_name = raw.distributor_push_target_name
|
||||
|
||||
return OrderUpsertInput(
|
||||
orderNo=platform_oid,
|
||||
outOrderNo=raw.source_platform_oid,
|
||||
courierSource=courier_source,
|
||||
courierName=raw.delivery_courier,
|
||||
courierNo=raw.courier_number,
|
||||
merchantId=raw.sub_name,
|
||||
countryRegion=normalize_country_region(raw.receiver_country, factory_name),
|
||||
factoryName=factory_name,
|
||||
systemName="RIIN",
|
||||
systemStatus=system_status,
|
||||
platformName=RIIN_PLATFORM_TYPE_MAP.get(raw.platform_type, str(raw.platform_type)) if raw.platform_type is not None else None,
|
||||
platformStatus=RIIN_PLATFORM_STATUS_MAP.get(raw.platform_order_status, raw.platform_order_status) if raw.platform_order_status else None,
|
||||
createdAt=safe_date(raw.order_create_time, f"RIIN order {platform_oid} orderCreateTime"),
|
||||
orderedAt=ordered_at,
|
||||
confirmedAt=confirmed_at,
|
||||
completedAt=completed_at,
|
||||
shipDeadline=ship_deadline,
|
||||
)
|
||||
|
||||
|
||||
def map_riin_goods(raw: Any, parent_order_no: str, parent_created_at: datetime) -> GoodsUpsertInput:
|
||||
product_code = raw.product_code
|
||||
item_name = product_code.split("-")[0] if product_code else None
|
||||
return GoodsUpsertInput(
|
||||
orderNo=parent_order_no,
|
||||
productionNo=raw.id,
|
||||
createdAt=parent_created_at,
|
||||
outOrderNo=raw.platform_oid,
|
||||
itemName=item_name,
|
||||
itemColor=raw.color_code,
|
||||
itemSize=raw.size_code,
|
||||
itemStyleNo=item_name,
|
||||
quantity=raw.num if raw.num is not None else 1,
|
||||
status=RIIN_GOODS_STATUS_MAP.get(raw.goods_status) if raw.goods_status else None,
|
||||
)
|
||||
|
||||
|
||||
# ---------- SDS 映射 ----------
|
||||
|
||||
def map_sds_channel_type(channel_type: Any) -> str | None:
|
||||
if not isinstance(channel_type, str):
|
||||
return None
|
||||
return SDS_CHANNEL_TYPE_LABEL_MAP.get(channel_type)
|
||||
|
||||
|
||||
def map_sds_system_status(status: int) -> str | None:
|
||||
return SDS_SYSTEM_STATUS_MAP.get(status)
|
||||
|
||||
|
||||
def map_sds_order(raw: Any, channel_type_map: dict[int, str | None], pre_courier: str | None = None) -> OrderUpsertInput:
|
||||
ordered_at = _parse(raw.gmt_pay, f"SDS order {raw.no} gmtPay")
|
||||
system_status = map_sds_system_status(raw.status) if raw.status is not None else None
|
||||
is_terminal = system_status in ("已完成", "已取消")
|
||||
completed_at: datetime | None = None
|
||||
if is_terminal and raw.finish_time and raw.finish_time > 0:
|
||||
completed_at = _from_epoch_ms(raw.finish_time)
|
||||
ship_deadline = _from_epoch_ms(raw.latest_ship_time) if raw.latest_ship_time else None
|
||||
|
||||
courier_source = pre_courier
|
||||
if courier_source is None and raw.logistics_id is not None:
|
||||
try:
|
||||
lid = int(raw.logistics_id)
|
||||
except (TypeError, ValueError):
|
||||
lid = None
|
||||
if lid is not None and lid in channel_type_map:
|
||||
courier_source = channel_type_map[lid]
|
||||
|
||||
courier_no: str | None = None
|
||||
if raw.parcel_list:
|
||||
first = raw.parcel_list[0]
|
||||
try:
|
||||
courier_no = first["logistics"]["carriage"]["no"]
|
||||
except (TypeError, KeyError, IndexError):
|
||||
courier_no = None
|
||||
|
||||
factory_name: str | None = None
|
||||
for item in raw.items or []:
|
||||
fname = getattr(getattr(item, "factory", None), "name", None)
|
||||
if not fname and isinstance(item, dict):
|
||||
fname = (item.get("factory") or {}).get("name")
|
||||
if fname:
|
||||
factory_name = fname
|
||||
break
|
||||
|
||||
area_name = raw.issuing_bay_area.name if raw.issuing_bay_area else None
|
||||
return OrderUpsertInput(
|
||||
orderNo=raw.no,
|
||||
outOrderNo=raw.out_order_no,
|
||||
courierSource=courier_source,
|
||||
courierName=raw.carriage_name,
|
||||
courierNo=courier_no,
|
||||
merchantId=raw.merchant.name if raw.merchant else None,
|
||||
countryRegion=normalize_country_region(area_name, factory_name),
|
||||
factoryName=factory_name,
|
||||
systemName="SDS",
|
||||
systemStatus=system_status,
|
||||
platformName=raw.merchant_store.platform_name if raw.merchant_store else None,
|
||||
platformStatus=raw.online_order_status_name,
|
||||
createdAt=safe_date(raw.gmt_created, f"SDS order {raw.no}"),
|
||||
orderedAt=ordered_at,
|
||||
confirmedAt=None,
|
||||
completedAt=completed_at,
|
||||
shipDeadline=ship_deadline,
|
||||
)
|
||||
|
||||
|
||||
def _parse_sds_style_no(name: str | None) -> str | None:
|
||||
if not name:
|
||||
return None
|
||||
parts = name.split("（）")
|
||||
if len(parts) < 3:
|
||||
return None
|
||||
subs = parts[2].split("-")
|
||||
if len(subs) == 2:
|
||||
return subs[0] or None
|
||||
if len(subs) >= 3:
|
||||
return subs[1] or None
|
||||
return None
|
||||
|
||||
|
||||
def map_sds_goods(raw: Any, parent_order_no: str, parent_created_at: datetime) -> GoodsUpsertInput:
|
||||
product_name = raw.product_name
|
||||
color_name = None
|
||||
block = raw.product_color_block
|
||||
if block is not None:
|
||||
color_name = getattr(block, "color_name", None) or (block.get("colorName") if isinstance(block, dict) else None)
|
||||
return GoodsUpsertInput(
|
||||
orderNo=parent_order_no,
|
||||
productionNo=raw.no or raw.id,
|
||||
createdAt=parent_created_at,
|
||||
outOrderNo=raw.out_order_item_id,
|
||||
itemName=product_name,
|
||||
itemColor=color_name,
|
||||
itemSize=raw.product_size,
|
||||
itemStyleNo=_parse_sds_style_no(product_name),
|
||||
quantity=raw.num if raw.num is not None else 1,
|
||||
status=SDS_GOODS_STATUS_MAP.get(raw.status) if raw.status is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def _parse(value: Any, context: str) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return _from_epoch_ms(value)
|
||||
try:
|
||||
return safe_date(value, context)
|
||||
except ValueError:
|
||||
return None
|
||||
Reference in New Issue
Block a user