init: Prefect ETL sync project (RIIN/SDS → PG) with 12 sync flows

flows/sds/recent_sync.py

@@ -0,0 +1,56 @@
+"""SDS 近期同步:拉取最近 N 天订单(默认 7,1~90),按 createdDate。
+
+对应 sds-recent-sync。from = startOfDay(now - N days),to = startOfDay(today)。
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from prefect import flow, task
+
+from flows.sds._pipeline import sync_sds_records
+from flows.shared import (
+    SyncDB, create_sds_client, fetch_sds_all_pages, format_date, get_channel_type_map,
+    refresh_channel_type_map, start_of_day,
+)
+
+
+@task(retries=3, retry_delay_seconds=[3, 6, 9])
+def extract_orders(params: dict) -> list:
+    client = create_sds_client()
+    try:
+        return fetch_sds_all_pages(client, params)
+    finally:
+        client.close()
+
+
+@task
+def upsert_records(raw_orders: list) -> dict:
+    client = create_sds_client()
+    try:
+        channel_map = get_channel_type_map(client)
+        return sync_sds_records(raw_orders, channel_map, SyncDB(),
+                                channel_refresh=lambda: refresh_channel_type_map(client))
+    finally:
+        client.close()
+
+
+@flow(name="sds-recent-sync", log_prints=True)
+def sds_recent_sync(recent_days: int = 7) -> dict:
+    days = min(max(recent_days, 1), 90)
+    now = datetime.now()
+    from_dt = start_of_day(now - timedelta(days=days))
+    to_dt = start_of_day(now)
+
+    params = {"created_date_begin": format_date(from_dt), "created_date_end": format_date(to_dt), "size": 500}
+    print(f"SDS 近期同步 近 {days} 天 {params['created_date_begin']} ~ {params['created_date_end']}")
+    raw = extract_orders(params)
+    stats = upsert_records(raw)
+    print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
+    return {"syncRange": {"from": params["created_date_begin"], "to": format_date(now)},
+            "recentDays": days, **stats}
+
+
+if __name__ == "__main__":
+    sds_recent_sync.serve(name="sds-recent-sync", cron="45 * * * *")