init: Prefect ETL sync project (RIIN/SDS → PG) with 12 sync flows
This commit is contained in:
yeuimu
@@ -0,0 +1 @@
|
||||
"""SDS 同步 flows:order_sync / incremental_sync / recent_sync / refresh_pending / refresh_100_prefix / verify。"""
|
||||
@@ -0,0 +1,58 @@
|
||||
"""SDS 同步公共管道:映射订单(含内联 items)→ upsert 到 PG。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
from flows.shared import resolve_courier_source
|
||||
from flows.shared.db import SyncDB
|
||||
from flows.shared.mappers import map_sds_goods, map_sds_order
|
||||
|
||||
|
||||
def map_sds_records(
|
||||
raw_orders: list[Any],
|
||||
channel_map: dict[int, str | None],
|
||||
channel_refresh: Callable[[], dict[int, str | None]] | None = None,
|
||||
skip_unpaid: bool = True,
|
||||
) -> tuple[list[Any], list[Any], int]:
|
||||
"""映射 SDS 订单+商品。返回 (order_inputs, goods_inputs, skipped_unpaid)。"""
|
||||
order_inputs: list[Any] = []
|
||||
goods_inputs: list[Any] = []
|
||||
refreshed: set[int] = set()
|
||||
skipped_unpaid = 0
|
||||
|
||||
for raw in raw_orders:
|
||||
courier = None
|
||||
if channel_refresh is not None:
|
||||
courier = resolve_courier_source(raw.logistics_id, channel_map, channel_refresh, refreshed)
|
||||
try:
|
||||
o = map_sds_order(raw, channel_map, courier)
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[SDS] 映射订单失败 {getattr(raw, 'no', '?')}: {err}")
|
||||
continue
|
||||
if skip_unpaid and o.systemStatus == "待付款":
|
||||
skipped_unpaid += 1
|
||||
continue
|
||||
order_inputs.append(o)
|
||||
for item in raw.items or []:
|
||||
try:
|
||||
goods_inputs.append(map_sds_goods(item, o.orderNo, o.createdAt))
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[SDS] 映射商品失败 order={o.orderNo}: {err}")
|
||||
|
||||
return order_inputs, goods_inputs, skipped_unpaid
|
||||
|
||||
|
||||
def sync_sds_records(
|
||||
raw_orders: list[Any], channel_map: dict[int, str | None], db: SyncDB,
|
||||
channel_refresh: Callable[[], dict[int, str | None]] | None = None,
|
||||
skip_unpaid: bool = True,
|
||||
) -> dict[str, int]:
|
||||
order_inputs, goods_inputs, skipped = map_sds_records(
|
||||
raw_orders, channel_map, channel_refresh, skip_unpaid
|
||||
)
|
||||
stats = db.sync_orders_and_goods(order_inputs, goods_inputs)
|
||||
stats["orderCount"] = len(order_inputs)
|
||||
stats["goodsCount"] = len(goods_inputs)
|
||||
stats["skippedUnpaid"] = skipped
|
||||
return stats
|
||||
@@ -0,0 +1,57 @@
|
||||
"""SDS 增量同步:从数据库最近下单时间增量拉取(无参,全自动)。
|
||||
|
||||
对应 sds-incremental-sync。from = floorToHour(MAX(orderedAt)),to = now,按 payDate。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from flows.sds._pipeline import sync_sds_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, fetch_sds_all_pages, floor_to_hour, format_date,
|
||||
get_channel_type_map, refresh_channel_type_map,
|
||||
)
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def extract_orders(params: dict) -> list:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
return fetch_sds_all_pages(client, params)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
channel_map = get_channel_type_map(client)
|
||||
return sync_sds_records(raw_orders, channel_map, SyncDB(),
|
||||
channel_refresh=lambda: refresh_channel_type_map(client))
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="sds-incremental-sync", log_prints=True)
|
||||
def sds_incremental_sync() -> dict:
|
||||
db = SyncDB()
|
||||
latest = db.get_latest_ordered_at("SDS")
|
||||
now = datetime.now()
|
||||
from_dt = floor_to_hour(latest) if latest else floor_to_hour(now)
|
||||
if from_dt >= now:
|
||||
from_dt = floor_to_hour(now)
|
||||
|
||||
params = {"pay_date_begin": format_date(from_dt), "pay_date_end": format_date(now), "size": 500}
|
||||
print(f"SDS 增量同步 {params['pay_date_begin']} ~ {params['pay_date_end']}")
|
||||
raw = extract_orders(params)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
|
||||
return {"syncRange": {"from": params["pay_date_begin"], "to": params["pay_date_end"]}, **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_incremental_sync.serve(name="sds-incremental-sync", cron="15 * * * *")
|
||||
@@ -0,0 +1,76 @@
|
||||
"""SDS 订单同步:按下单时间窗口(+筛选)拉取订单与商品。
|
||||
|
||||
对应 sds-order-sync。主时间维度 payDate(下单时间)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from flows.sds._pipeline import sync_sds_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, fetch_sds_all_pages, get_channel_type_map, refresh_channel_type_map,
|
||||
)
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def extract_orders(params: dict[str, Any]) -> list:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
return fetch_sds_all_pages(client, params)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
channel_map = get_channel_type_map(client)
|
||||
return sync_sds_records(raw_orders, channel_map, SyncDB(),
|
||||
channel_refresh=lambda: refresh_channel_type_map(client))
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="sds-order-sync", log_prints=True)
|
||||
def sds_order_sync(
|
||||
pay_date_begin: str,
|
||||
pay_date_end: str,
|
||||
order_status_code: str | None = None,
|
||||
factory_ids: list[int] | None = None,
|
||||
merchant_ids: list[int] | None = None,
|
||||
logistics_ids: list[int] | None = None,
|
||||
issuing_bay_area_id: int | None = None,
|
||||
keywords: str | None = None,
|
||||
page_size: int = 500,
|
||||
) -> dict[str, Any]:
|
||||
params: dict[str, Any] = {
|
||||
"pay_date_begin": pay_date_begin,
|
||||
"pay_date_end": pay_date_end,
|
||||
"size": page_size,
|
||||
}
|
||||
if order_status_code is not None:
|
||||
params["order_status_code"] = order_status_code
|
||||
if factory_ids:
|
||||
params["factory_ids"] = factory_ids
|
||||
if merchant_ids:
|
||||
params["merchant_ids"] = merchant_ids
|
||||
if logistics_ids:
|
||||
params["logistics_ids"] = logistics_ids
|
||||
if issuing_bay_area_id is not None:
|
||||
params["issuing_bay_area_id"] = issuing_bay_area_id
|
||||
if keywords:
|
||||
params["keywords"] = keywords
|
||||
|
||||
print(f"SDS 订单同步 {pay_date_begin} ~ {pay_date_end}")
|
||||
raw = extract_orders(params)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} / 跳过待付款 {stats.get('skippedUnpaid',0)}")
|
||||
return {"syncRange": {"from": pay_date_begin, "to": pay_date_end}, **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_order_sync.serve(name="sds-order-sync")
|
||||
@@ -0,0 +1,56 @@
|
||||
"""SDS 近期同步:拉取最近 N 天订单(默认 7,1~90),按 createdDate。
|
||||
|
||||
对应 sds-recent-sync。from = startOfDay(now - N days),to = startOfDay(today)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from flows.sds._pipeline import sync_sds_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, fetch_sds_all_pages, format_date, get_channel_type_map,
|
||||
refresh_channel_type_map, start_of_day,
|
||||
)
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def extract_orders(params: dict) -> list:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
return fetch_sds_all_pages(client, params)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
channel_map = get_channel_type_map(client)
|
||||
return sync_sds_records(raw_orders, channel_map, SyncDB(),
|
||||
channel_refresh=lambda: refresh_channel_type_map(client))
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="sds-recent-sync", log_prints=True)
|
||||
def sds_recent_sync(recent_days: int = 7) -> dict:
|
||||
days = min(max(recent_days, 1), 90)
|
||||
now = datetime.now()
|
||||
from_dt = start_of_day(now - timedelta(days=days))
|
||||
to_dt = start_of_day(now)
|
||||
|
||||
params = {"created_date_begin": format_date(from_dt), "created_date_end": format_date(to_dt), "size": 500}
|
||||
print(f"SDS 近期同步 近 {days} 天 {params['created_date_begin']} ~ {params['created_date_end']}")
|
||||
raw = extract_orders(params)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
|
||||
return {"syncRange": {"from": params["created_date_begin"], "to": format_date(now)},
|
||||
"recentDays": days, **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_recent_sync.serve(name="sds-recent-sync", cron="45 * * * *")
|
||||
@@ -0,0 +1,104 @@
|
||||
"""SDS 100前缀刷新:备货中/待确认且 productionNo 以 100 开头的订单,工厂分配后用新数据替换。
|
||||
|
||||
对应 sds-refresh-100-prefix。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from api.sds.models import OrderPageRequest
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, get_channel_type_map,
|
||||
refresh_channel_type_map, resolve_courier_source,
|
||||
)
|
||||
from flows.shared.mappers import map_sds_goods, map_sds_order
|
||||
|
||||
TARGET_STATUSES = ["备货中", "待确认"]
|
||||
KEYWORDS_BATCH = 200
|
||||
|
||||
|
||||
def _has_100_prefix_item(raw) -> bool:
|
||||
for item in raw.items or []:
|
||||
prod_no = (item.no or item.id) if hasattr(item, "no") else None
|
||||
if prod_no and str(prod_no).startswith("100"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def fetch_remote(order_nos: list[str]) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
remote: dict[str, object] = {}
|
||||
for i in range(0, len(order_nos), KEYWORDS_BATCH):
|
||||
batch = order_nos[i : i + KEYWORDS_BATCH]
|
||||
resp = client.get_orders(OrderPageRequest(keywords="\n".join(batch), all=True))
|
||||
for o in resp.records:
|
||||
if o.no:
|
||||
remote[o.no] = o
|
||||
return remote
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(remote_map: dict, target_order_nos: list[str]) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
channel_map = get_channel_type_map(client)
|
||||
refreshed: set[int] = set()
|
||||
|
||||
order_inputs: list = []
|
||||
goods_inputs: list = []
|
||||
skipped_pending = skipped_unpaid = 0
|
||||
|
||||
for no in target_order_nos:
|
||||
raw = remote_map.get(no)
|
||||
if not raw:
|
||||
continue
|
||||
if _has_100_prefix_item(raw):
|
||||
skipped_pending += 1
|
||||
continue
|
||||
courier = resolve_courier_source(raw.logistics_id, channel_map,
|
||||
lambda: refresh_channel_type_map(client), refreshed)
|
||||
try:
|
||||
o = map_sds_order(raw, channel_map, courier)
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[SDS 100] 映射失败 {no}: {err}")
|
||||
continue
|
||||
if o.systemStatus == "待付款":
|
||||
skipped_unpaid += 1
|
||||
continue
|
||||
order_inputs.append(o)
|
||||
for item in raw.items or []:
|
||||
try:
|
||||
goods_inputs.append(map_sds_goods(item, o.orderNo, o.createdAt))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
stats = SyncDB().sync_orders_and_goods(order_inputs, goods_inputs)
|
||||
stats["skippedStillPending"] = skipped_pending
|
||||
stats["skippedUnpaid"] = skipped_unpaid
|
||||
return stats
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="sds-refresh-100-prefix", log_prints=True)
|
||||
def sds_refresh_100_prefix() -> dict:
|
||||
db = SyncDB()
|
||||
targets = db.get_orders_with_100_prefix("SDS", TARGET_STATUSES)
|
||||
order_nos = list({o for o, _ in targets})
|
||||
print(f"SDS 100前缀目标订单 {len(order_nos)} 个")
|
||||
if not order_nos:
|
||||
return {"targetCount": 0, "refreshed": 0}
|
||||
|
||||
remote = fetch_remote(order_nos)
|
||||
stats = upsert_records(remote, order_nos)
|
||||
print(f"完成: 刷新 {stats['orderCount']} / 仍待分配 {stats.get('skippedStillPending',0)} / 待付款 {stats.get('skippedUnpaid',0)}")
|
||||
return {"targetCount": len(order_nos), **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_refresh_100_prefix.serve(name="sds-refresh-100-prefix", cron="30 */2 * * *")
|
||||
@@ -0,0 +1,61 @@
|
||||
"""SDS 刷新未完结:查 DB 未完结订单号,按 keywords 批量刷新。
|
||||
|
||||
对应 sds-refresh-pending。终态:已完成 / 已取消。注意不跳过待付款。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from api.sds.models import OrderPageRequest
|
||||
from flows.sds._pipeline import sync_sds_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, get_channel_type_map, refresh_channel_type_map,
|
||||
)
|
||||
|
||||
SDS_FINISHED_STATUSES = ["已完成", "已取消"]
|
||||
KEYWORDS_BATCH = 500
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=[3, 6, 9])
|
||||
def fetch_pending(order_nos: list[str]) -> list:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
all_raw: list = []
|
||||
for i in range(0, len(order_nos), KEYWORDS_BATCH):
|
||||
batch = order_nos[i : i + KEYWORDS_BATCH]
|
||||
resp = client.get_orders(OrderPageRequest(keywords="\n".join(batch), all=True))
|
||||
all_raw.extend(resp.records)
|
||||
return all_raw
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@task
|
||||
def upsert_records(raw_orders: list) -> dict:
|
||||
client = create_sds_client()
|
||||
try:
|
||||
channel_map = get_channel_type_map(client)
|
||||
return sync_sds_records(raw_orders, channel_map, SyncDB(),
|
||||
channel_refresh=lambda: refresh_channel_type_map(client),
|
||||
skip_unpaid=False)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
@flow(name="sds-refresh-pending", log_prints=True)
|
||||
def sds_refresh_pending() -> dict:
|
||||
db = SyncDB()
|
||||
order_nos = db.get_pending_orders("SDS", SDS_FINISHED_STATUSES)
|
||||
print(f"SDS 未完结订单 {len(order_nos)} 个")
|
||||
if not order_nos:
|
||||
return {"pendingCount": 0, "matchedCount": 0}
|
||||
|
||||
raw = fetch_pending(order_nos)
|
||||
stats = upsert_records(raw)
|
||||
print(f"完成: 匹配 {len(raw)} | {stats}")
|
||||
return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_refresh_pending.serve(name="sds-refresh-pending", cron="0 */2 * * *")
|
||||
@@ -0,0 +1,104 @@
|
||||
"""SDS 数据核验:逐天比对远程/本地按状态计数(排除"售后"),有差异则拉取修复。
|
||||
|
||||
对应 sds-verify。mode: recent(近 N 天,不含今天)/ range(指定区间)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from prefect import flow, task
|
||||
|
||||
from api.sds.models import OrderQueryParams
|
||||
from flows.sds._pipeline import sync_sds_records
|
||||
from flows.shared import (
|
||||
SyncDB, create_sds_client, fetch_sds_all_pages, format_date, get_channel_type_map,
|
||||
refresh_channel_type_map, start_of_day,
|
||||
)
|
||||
|
||||
|
||||
def _parse_days(mode: str, recent_days: int, date_range: str | None) -> list[datetime]:
|
||||
today = start_of_day(datetime.now())
|
||||
if mode == "range" and date_range:
|
||||
parts = [p.strip() for p in date_range.split("~")]
|
||||
from_dt = datetime.fromisoformat(parts[0])
|
||||
to_dt = datetime.fromisoformat(parts[1]) if len(parts) > 1 and parts[1] else from_dt
|
||||
days = []
|
||||
cur = from_dt
|
||||
while cur <= to_dt:
|
||||
days.append(cur)
|
||||
cur += timedelta(days=1)
|
||||
return days
|
||||
n = min(max(recent_days, 1), 90)
|
||||
return [today - timedelta(days=i) for i in range(n, 0, -1)]
|
||||
|
||||
|
||||
def _diff(remote_stats: list, local: dict[str, int]) -> tuple[dict[str, int], bool]:
|
||||
diff: dict[str, int] = {}
|
||||
has_diff = False
|
||||
statuses = {getattr(s, "name", None) for s in remote_stats} | set(local)
|
||||
statuses.discard("售后")
|
||||
statuses.discard(None)
|
||||
for status in statuses:
|
||||
remote = next((getattr(s, "orderNum", 0) for s in remote_stats if getattr(s, "name", None) == status), 0)
|
||||
local_n = local.get(status, 0)
|
||||
if remote != local_n:
|
||||
diff[status] = local_n - remote
|
||||
has_diff = True
|
||||
return diff, has_diff
|
||||
|
||||
|
||||
@flow(name="sds-verify", log_prints=True)
|
||||
def sds_verify(mode: str = "recent", recent_days: int = 1, date_range: str | None = None) -> dict:
|
||||
days = _parse_days(mode, recent_days, date_range)
|
||||
db = SyncDB()
|
||||
client = create_sds_client()
|
||||
channel_map = get_channel_type_map(client)
|
||||
|
||||
totals = {k: 0 for k in ("ordersAdded", "ordersUpdated", "goodsAdded", "goodsUpdated")}
|
||||
days_with_diff = 0
|
||||
day_reports = []
|
||||
|
||||
try:
|
||||
for day in days:
|
||||
day_str = format_date(day).split(" ")[0]
|
||||
day_start = start_of_day(day)
|
||||
day_end = day_start + timedelta(days=1)
|
||||
|
||||
remote_stats = client.get_status(
|
||||
OrderQueryParams(pay_date_begin=format_date(day_start), pay_date_end=format_date(day_end))
|
||||
)
|
||||
remote_total = sum(getattr(s, "orderNum", 0) or 0 for s in remote_stats)
|
||||
local_counts = db.get_order_count_by_status("SDS", day_start, day_end)
|
||||
diff, has_diff = _diff(remote_stats, local_counts)
|
||||
|
||||
print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}"
|
||||
+ (f" 差异 {diff}" if has_diff else " 一致,跳过"))
|
||||
if not has_diff:
|
||||
day_reports.append({"date": day_str, "diff": diff, "ok": True})
|
||||
continue
|
||||
|
||||
days_with_diff += 1
|
||||
raw = fetch_sds_all_pages(
|
||||
client,
|
||||
{"pay_date_begin": format_date(day_start), "pay_date_end": format_date(day_end), "size": 500},
|
||||
)
|
||||
stats = sync_sds_records(raw, channel_map, db,
|
||||
channel_refresh=lambda: refresh_channel_type_map(client))
|
||||
totals["ordersAdded"] += stats["ordersCreated"]
|
||||
totals["ordersUpdated"] += stats["ordersUpdated"]
|
||||
totals["goodsAdded"] += stats["goodsCreated"]
|
||||
totals["goodsUpdated"] += stats["goodsUpdated"]
|
||||
day_reports.append({"date": day_str, "diff": diff, "ok": False,
|
||||
"orders": stats["orderCount"], "goods": stats["goodsCount"]})
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
return {
|
||||
"verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}},
|
||||
"syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""},
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sds_verify.serve(name="sds-verify", cron="0 7 * * *")
|
||||
Reference in New Issue
Block a user