init: Prefect ETL sync project (RIIN/SDS → PG) with 12 sync flows

flows/riin/__init__.py

@@ -0,0 +1 @@
+"""RIIN 同步 flows:order_sync / incremental_sync / recent_sync / refresh_pending / by_orders / verify。"""

flows/riin/_pipeline.py

@@ -0,0 +1,45 @@
+"""RIIN 同步公共管道:映射订单 → 并发拉商品 → upsert 到 PG。"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from flows.shared import fetch_goods_concurrent
+from flows.shared.db import SyncDB
+from flows.shared.mappers import map_riin_goods, map_riin_order
+from api import RiinDisClient
+
+
+def sync_riin_records(raw_orders: list[Any], client: RiinDisClient, db: SyncDB) -> dict[str, int]:
+    order_inputs: list[Any] = []
+    order_meta: dict[str, tuple[str, Any]] = {}  # raw.id -> (orderNo, createdAt)
+    goods_failed = 0
+
+    for raw in raw_orders:
+        try:
+            o = map_riin_order(raw)
+        except Exception as err:  # noqa: BLE001
+            print(f"[RIIN] 映射订单失败 {getattr(raw, 'platform_oid', '?')}: {err}")
+            continue
+        order_inputs.append(o)
+        if raw.id:
+            order_meta[raw.id] = (o.orderNo, o.createdAt)
+
+    goods_inputs: list[Any] = []
+    if order_meta:
+        goods_map = fetch_goods_concurrent(client, list(order_meta.keys()))
+        for oid, (order_no, created_at) in order_meta.items():
+            items = goods_map.get(oid, [])
+            if not items:
+                goods_failed += 1
+            for g in items:
+                try:
+                    goods_inputs.append(map_riin_goods(g, order_no, created_at))
+                except Exception as err:  # noqa: BLE001
+                    print(f"[RIIN] 映射商品失败 order={order_no}: {err}")
+
+    stats = db.sync_orders_and_goods(order_inputs, goods_inputs)
+    stats["orderCount"] = len(order_inputs)
+    stats["goodsCount"] = len(goods_inputs)
+    stats["goodsFetchFailed"] = goods_failed
+    return stats

flows/riin/by_orders.py

@@ -0,0 +1,59 @@
+"""RIIN 指定单号同步:通过订单号列表(逗号/换行分隔)拉取指定订单。
+
+对应 riin-by-orders。
+"""
+
+from __future__ import annotations
+
+import re
+
+from prefect import flow, task
+
+from api.riin.models import OrderPageRequest
+from flows.riin._pipeline import sync_riin_records
+from flows.shared import SyncDB, create_riin_client
+
+
+def _split_order_nos(text: str) -> list[str]:
+    seen: dict[str, None] = {}
+    for part in re.split(r"[\n\r,，\s]+", text or ""):
+        part = part.strip()
+        if part and part not in seen:
+            seen[part] = None
+    return list(seen.keys())
+
+
+@task(retries=3, retry_delay_seconds=[3, 6, 9])
+def fetch_by_orders(order_nos: list[str]) -> list:
+    client = create_riin_client()
+    try:
+        resp = client.get_orders(OrderPageRequest(platform_oid_list=",".join(order_nos), all=True))
+        return list(resp.data.records)
+    finally:
+        client.close()
+
+
+@task
+def upsert_records(raw_orders: list) -> dict:
+    client = create_riin_client()
+    try:
+        return sync_riin_records(raw_orders, client, SyncDB())
+    finally:
+        client.close()
+
+
+@flow(name="riin-by-orders-sync", log_prints=True)
+def riin_by_orders_sync(order_nos: str) -> dict:
+    nos = _split_order_nos(order_nos)
+    print(f"RIIN 指定单号同步 输入 {len(nos)} 个")
+    if not nos:
+        return {"inputCount": 0, "matchedCount": 0}
+
+    raw = fetch_by_orders(nos)
+    stats = upsert_records(raw)
+    print(f"完成: 匹配 {len(raw)} | {stats}")
+    return {"inputCount": len(nos), "matchedCount": len(raw), **stats}
+
+
+if __name__ == "__main__":
+    riin_by_orders_sync.serve(name="riin-by-orders-sync")

flows/riin/incremental_sync.py

@@ -0,0 +1,60 @@
+"""RIIN 增量同步:从数据库最近下单时间增量拉取(无参,全自动)。
+
+对应 riin-incremental-sync。from = floorToHour(MAX(orderedAt)),to = now。
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from prefect import flow, task
+
+from flows.riin._pipeline import sync_riin_records
+from flows.shared import (
+    SyncDB, create_riin_client, fetch_riin_all_pages, floor_to_hour, format_date,
+)
+
+
+@task(retries=3, retry_delay_seconds=[3, 6, 9])
+def extract_orders(params: dict) -> list:
+    client = create_riin_client()
+    try:
+        return fetch_riin_all_pages(client, params)
+    finally:
+        client.close()
+
+
+@task
+def upsert_records(raw_orders: list) -> dict:
+    client = create_riin_client()
+    try:
+        return sync_riin_records(raw_orders, client, SyncDB())
+    finally:
+        client.close()
+
+
+@flow(name="riin-incremental-sync", log_prints=True)
+def riin_incremental_sync() -> dict:
+    db = SyncDB()
+    latest = db.get_latest_ordered_at("RIIN")
+    now = datetime.now()
+    from_dt = floor_to_hour(latest) if latest else floor_to_hour(now)
+    to_dt = now
+    if from_dt >= to_dt:
+        from_dt = floor_to_hour(now)
+
+    params = {
+        "time_type": 3,
+        "start_time": format_date(from_dt),
+        "end_time": format_date(to_dt),
+        "page_size": 200,
+    }
+    print(f"RIIN 增量同步 {params['start_time']} ~ {params['end_time']}")
+    raw = extract_orders(params)
+    stats = upsert_records(raw)
+    print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
+    return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, **stats}
+
+
+if __name__ == "__main__":
+    riin_incremental_sync.serve(name="riin-incremental-sync", cron="0 * * * *")

flows/riin/order_sync.py

@@ -0,0 +1,77 @@
+"""RIIN 订单同步:按时间窗口(+可选筛选)拉取订单与商品。
+
+对应 riin-order-sync。timeType 固定 3(商家推送时间)。
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from prefect import flow, task
+
+from api.riin.models import OrderPageRequest
+from flows.riin._pipeline import sync_riin_records
+from flows.shared import SyncDB, create_riin_client, fetch_riin_all_pages, format_date
+
+
+@task(retries=3, retry_delay_seconds=[3, 6, 9])
+def extract_orders(params: dict[str, Any]) -> list[Any]:
+    client = create_riin_client()
+    try:
+        return fetch_riin_all_pages(client, params)
+    finally:
+        client.close()
+
+
+@task
+def upsert_records(raw_orders: list[Any]) -> dict[str, int]:
+    client = create_riin_client()
+    try:
+        return sync_riin_records(raw_orders, client, SyncDB())
+    finally:
+        client.close()
+
+
+@flow(name="riin-order-sync", log_prints=True)
+def riin_order_sync(
+    order_date_begin: str,
+    order_date_end: str,
+    platform_type: int | None = None,
+    order_status: int | None = None,
+    factory_ids: list[str] | None = None,
+    express_source: int | None = None,
+    urgent_order: int | None = None,
+    has_refund: bool | None = None,
+    page_size: int = 200,
+) -> dict[str, Any]:
+    if not order_date_begin or not order_date_end:
+        raise ValueError("order_date_begin / order_date_end 为必填")
+
+    params: dict[str, Any] = {
+        "time_type": 3,
+        "start_time": order_date_begin,
+        "end_time": order_date_end,
+        "page_size": page_size,
+    }
+    if platform_type is not None:
+        params["platform_type"] = platform_type
+    if order_status is not None:
+        params["order_status"] = order_status
+    if factory_ids:
+        params["factory_ids"] = factory_ids
+    if express_source is not None:
+        params["express_source"] = express_source
+    if urgent_order is not None:
+        params["urgent_order"] = urgent_order
+    if has_refund is not None:
+        params["has_refund"] = has_refund
+
+    print(f"RIIN 订单同步 {order_date_begin} ~ {order_date_end}")
+    raw = extract_orders(params)
+    stats = upsert_records(raw)
+    print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} | {stats}")
+    return {"syncRange": {"from": order_date_begin, "to": order_date_end}, **stats}
+
+
+if __name__ == "__main__":
+    riin_order_sync.serve(name="riin-order-sync")

flows/riin/recent_sync.py

@@ -0,0 +1,56 @@
+"""RIIN 近期同步:拉取最近 N 天订单(默认 7 天,1~90)。
+
+对应 riin-recent-sync。from = startOfDay(now - N days),to = now。
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from prefect import flow, task
+
+from flows.riin._pipeline import sync_riin_records
+from flows.shared import (
+    SyncDB, create_riin_client, fetch_riin_all_pages, format_date, start_of_day,
+)
+
+
+@task(retries=3, retry_delay_seconds=[3, 6, 9])
+def extract_orders(params: dict) -> list:
+    client = create_riin_client()
+    try:
+        return fetch_riin_all_pages(client, params)
+    finally:
+        client.close()
+
+
+@task
+def upsert_records(raw_orders: list) -> dict:
+    client = create_riin_client()
+    try:
+        return sync_riin_records(raw_orders, client, SyncDB())
+    finally:
+        client.close()
+
+
+@flow(name="riin-recent-sync", log_prints=True)
+def riin_recent_sync(recent_days: int = 7) -> dict:
+    days = min(max(recent_days, 1), 90)
+    now = datetime.now()
+    from_dt = start_of_day(now - timedelta(days=days))
+
+    params = {
+        "time_type": 3,
+        "start_time": format_date(from_dt),
+        "end_time": format_date(now),
+        "page_size": 200,
+    }
+    print(f"RIIN 近期同步 近 {days} 天 {params['start_time']} ~ {params['end_time']}")
+    raw = extract_orders(params)
+    stats = upsert_records(raw)
+    print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
+    return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, "recentDays": days, **stats}
+
+
+if __name__ == "__main__":
+    riin_recent_sync.serve(name="riin-recent-sync", cron="30 * * * *")

flows/riin/refresh_pending.py

@@ -0,0 +1,58 @@
+"""RIIN 刷新未完结:查 DB 中未完结订单号,按 platformOidList 批量(500)刷新。
+
+对应 riin-refresh-pending。终态:已退款 / 已关闭 / 已发货。
+"""
+
+from __future__ import annotations
+
+from prefect import flow, task
+
+from api.riin.models import OrderPageRequest
+from flows.riin._pipeline import sync_riin_records
+from flows.shared import SyncDB, create_riin_client
+
+RIIN_FINISHED_STATUSES = ["已退款", "已关闭", "已发货"]
+ORDER_BATCH = 500
+
+
+@task(retries=3, retry_delay_seconds=[3, 6, 9])
+def fetch_pending(order_nos: list[str]) -> list:
+    client = create_riin_client()
+    try:
+        all_raw: list = []
+        for i in range(0, len(order_nos), ORDER_BATCH):
+            batch = order_nos[i : i + ORDER_BATCH]
+            resp = client.get_orders(
+                OrderPageRequest(platform_oid_list=",".join(batch), all=True)
+            )
+            all_raw.extend(resp.data.records)
+        return all_raw
+    finally:
+        client.close()
+
+
+@task
+def upsert_records(raw_orders: list) -> dict:
+    client = create_riin_client()
+    try:
+        return sync_riin_records(raw_orders, client, SyncDB())
+    finally:
+        client.close()
+
+
+@flow(name="riin-refresh-pending", log_prints=True)
+def riin_refresh_pending() -> dict:
+    db = SyncDB()
+    order_nos = db.get_pending_orders("RIIN", RIIN_FINISHED_STATUSES)
+    print(f"RIIN 未完结订单 {len(order_nos)} 个")
+    if not order_nos:
+        return {"pendingCount": 0, "matchedCount": 0}
+
+    raw = fetch_pending(order_nos)
+    stats = upsert_records(raw)
+    print(f"完成: 匹配 {len(raw)} | {stats}")
+    return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats}
+
+
+if __name__ == "__main__":
+    riin_refresh_pending.serve(name="riin-refresh-pending", cron="0 */3 * * *")

flows/riin/verify.py

@@ -0,0 +1,126 @@
+"""RIIN 数据核验:逐天比对远程/本地按状态计数,有差异则拉取修复。
+
+对应 riin-verify。mode: recent(近 N 天,不含今天)/ range(指定区间)。
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from prefect import flow, task
+
+from api.riin.models import OrderPageRequest
+from flows.riin._pipeline import sync_riin_records
+from flows.shared import (
+    SyncDB, create_riin_client, fetch_riin_all_pages, format_date, start_of_day,
+)
+
+RIIN_COUNT_STATUS_MAP = {
+    "assignedSuceessed": "已分配",
+    "factoryAuditing": "工厂审核中",
+    "inProducing": "生产中",
+    "shipped": "已发货",
+    "closed": "已关闭",
+    "refunding": "退款中",
+    "refunded": "已退款",
+    "rejecting": "拒审中",
+    "rejectedToShop": "已退回店铺",
+}
+
+
+def _parse_days(mode: str, recent_days: int, date_range: str | None) -> list[datetime]:
+    today = start_of_day(datetime.now())
+    if mode == "range" and date_range:
+        parts = [p.strip() for p in date_range.split("~")]
+        from_dt = datetime.fromisoformat(parts[0])
+        to_dt = datetime.fromisoformat(parts[1]) if len(parts) > 1 and parts[1] else from_dt
+        days = []
+        cur = from_dt
+        while cur <= to_dt:
+            days.append(cur)
+            cur += timedelta(days=1)
+        return days
+    days_n = min(max(recent_days, 1), 90)
+    return [today - timedelta(days=i) for i in range(days_n, 0, -1)]
+
+
+def _count_to_status_map(data: dict) -> dict[str, int]:
+    out: dict[str, int] = {}
+    for key, count in data.items():
+        if key.startswith("dayRange"):
+            continue
+        label = RIIN_COUNT_STATUS_MAP.get(key)
+        if label and count and count > 0:
+            out[label] = count
+    return out
+
+
+def _diff(remote: dict[str, int], local: dict[str, int]) -> tuple[dict[str, int], bool]:
+    diff: dict[str, int] = {}
+    has_diff = False
+    for status in set(remote) | set(local):
+        r = remote.get(status, 0)
+        l = local.get(status, 0)
+        if r != l:
+            diff[status] = l - r
+            has_diff = True
+    return diff, has_diff
+
+
+@flow(name="riin-verify", log_prints=True)
+def riin_verify(mode: str = "recent", recent_days: int = 1, date_range: str | None = None) -> dict:
+    days = _parse_days(mode, recent_days, date_range)
+    db = SyncDB()
+    client = create_riin_client()
+
+    totals = {k: 0 for k in ("ordersAdded", "ordersUpdated", "ordersRemoved",
+                              "goodsAdded", "goodsUpdated", "goodsRemoved")}
+    days_with_diff = 0
+    day_reports = []
+
+    try:
+        for day in days:
+            day_str = format_date(day).split(" ")[0]
+            day_start = start_of_day(day)
+            day_end = day_start + timedelta(days=1)
+
+            count_resp = client.get_order_count(
+                OrderPageRequest(time_type=3, start_time=format_date(day_start), end_time=format_date(day_end))
+            )
+            remote_map = _count_to_status_map(count_resp.data.model_dump(by_alias=False))
+            remote_total = sum(remote_map.values())
+            local_counts = db.get_order_count_by_status("RIIN", day_start, day_end)
+            local_goods = db.get_local_goods_count("RIIN", day_start, day_end)
+            diff, has_diff = _diff(remote_map, local_counts)
+
+            print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}"
+                  + (f" 差异 {diff}" if has_diff else " 一致,跳过"))
+            if not has_diff:
+                day_reports.append({"date": day_str, "diff": diff, "ok": True})
+                continue
+
+            days_with_diff += 1
+            raw_orders = fetch_riin_all_pages(
+                client,
+                {"time_type": 3, "start_time": format_date(day_start), "end_time": format_date(day_end), "page_size": 200},
+            )
+            stats = sync_riin_records(raw_orders, client, db)
+            for k in ("ordersCreated", "ordersUpdated", "goodsCreated", "goodsUpdated"):
+                pass
+            totals["ordersAdded"] += stats["ordersCreated"]
+            totals["ordersUpdated"] += stats["ordersUpdated"]
+            totals["goodsAdded"] += stats["goodsCreated"]
+            totals["goodsUpdated"] += stats["goodsUpdated"]
+            day_reports.append({"date": day_str, "diff": diff, "ok": False,
+                                "orders": stats["orderCount"], "goods": stats["goodsCount"]})
+    finally:
+        client.close()
+
+    return {
+        "verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}},
+        "syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""},
+    }
+
+
+if __name__ == "__main__":
+    riin_verify.serve(name="riin-verify", cron="0 6 * * *")