init: Prefect ETL sync project (RIIN/SDS → PG) with 12 sync flows

This commit is contained in:
yeuimu
2026-07-23 08:13:09 +00:00
commit 4cba1da4e1
41 changed files with 6219 additions and 0 deletions
+9
View File
@@ -0,0 +1,9 @@
# PostgreSQL (ETL 目标库) - 复制为 .env 并填入真实值
DB_HOST=192.168.124.92
DB_PORT=5432
DB_USER=postgres
DB_PASSWORD=your-password-here
DB_NAME=inkreach-dmp
# 完整连接串(供 SQLAlchemy / prefect 使用)
DB_DSN=postgresql://postgres:your-password-here@192.168.124.92:5432/inkreach-dmp
+8
View File
@@ -0,0 +1,8 @@
.env
__pycache__/
*.pyc
.venv/
*.csv
external_projects/
*.log
.prefect/
+73
View File
@@ -0,0 +1,73 @@
# AGENTS.md
ETL 工作流项目,使用 [Prefect](https://docs.prefect.org.cn/v3/get-started) 编排。
任务:REST API → PostgreSQL(Extract / Transform / Load)。
## 设计理念(简述)
- **高内聚、低耦合、关注点分离**。
- `api/` 是纯客户端层(httpx + pydantic,**禁止 import prefect**),封装 RIIN/SDS 两个外部系统的 REST API,可独立复用/测试。
- `flows/` 是编排层(依赖 `api/`,使用 prefect),负责重试/缓存/并发/调度/密钥。
- 共享行为(token 注入、401 重登、分页、批量、缓存)放 `api/_common/`,用组合式 helper,不过度抽象。
- 新增系统是加法式:加一个 `api/<系统>/` + 一个 flow,不碰旧代码。
- 详细架构说明见 [`docs/dev/systematic-design.md`](docs/dev/systematic-design.md)。
## 依赖管理:使用 uv(不要用 requirements.txt)
本项目用 [uv](https://docs.astral.sh/uv/) 管理依赖,锁定文件为 `uv.lock`。
**没有 requirements.txt,也不要新建。**
```bash
uv sync # 按 uv.lock 安装依赖到 .venv(首次/拉取代码后执行)
uv add <package> # 添加依赖(自动写入 pyproject.toml + 更新 uv.lock)
uv add --dev <package> # 添加开发依赖
uv remove <package> # 移除依赖
uv lock --upgrade # 升级所有依赖的锁定版本
uv run python <file> # 在项目环境中运行(无需手动激活 venv)
uv run prefect <cmd> # 运行 prefect CLI
```
- Python 版本固定为 `>=3.12`(见 `pyproject.toml`)。
- 所有命令都用 `uv run ...` 前缀,**不要**直接用系统 `python` / `pip`。
- 提交代码时一并提交 `pyproject.toml` 和 `uv.lock`;**不要提交 `.venv/`**。
## 运行环境
- **Prefect server**:自托管,通过 Docker Compose 运行,数据持久化。
```bash
docker compose up -d # 启动 server(UI: http://192.168.124.92:4200)
docker compose ps # 查看状态
docker compose logs -f # 查看日志
docker compose down # 停止
```
- **API 地址**:`http://localhost:4200/api`(本机)/ `http://192.168.124.92:4200`(局域网)。
- 客户端已配置:`uv run prefect config set PREFECT_API_URL="http://localhost:4200/api"`。
## 密钥管理(标准做法)
- **不要把密钥写进代码或 pyproject.toml**。
- 数据库凭据用 Prefect Block 加密存储,Block 名:`pg-inkreach-dmp`(`SqlAlchemyConnector` 类型)。
```python
from prefect_sqlalchemy import SqlAlchemyConnector
connector = SqlAlchemyConnector.load("pg-inkreach-dmp")
engine = connector.get_engine()
```
- RIIN / SDS 登录凭据用 `Secret` Block 加密存储,值为 JSON:`{"username":"...","password":"..."}`。
Block 名:`riin-credentials`、`sds-credentials`。flow 在 extract task 内部读取,不外泄。
- 本地开发备用:`.env`(已 gitignore)含 `DB_DSN`,加载用 `python-dotenv` 或 `os.environ`。
## 编写/运行工作流
- 同步任务(对齐 inkreach-bi-auth-mng 的 sync tasks)放在 `flows/riin/` 和 `flows/sds/`,每系统各 6 个:
- `order_sync`(按时间窗口)、`incremental_sync`(按 DB 最近下单时间增量,无参)、
`recent_sync`(近 N 天)、`refresh_pending`(刷新未完结)、`by_orders`(RIIN 指定单号)/ `refresh_100_prefix`(SDS 100 前缀)、`verify`(逐天核验修复)。
- 共享层在 `flows/shared/`:`mappers.py`(RIIN/SDS→统一 Order/Goods)、`db.py`(PG upsert `SyncDB`)、`clients.py`(客户端工厂+顺序分页+重试+渠道字典)。
- 运行单个 flow:`uv run python flows/riin/<file>.py`(或 `flows/sds/<file>.py`)
- 部署调度:flow 末尾 `flow.serve(name=..., cron=...)`,保持 serve 进程运行。
- 写入目标:PG 的 `Orders` / `Goods` / `Merchant` 表(主键 `Orders=("orderNo","createdAt")`、`Goods=("orderNo","productionNo")`),`createdAt` 写入前以库中已有值覆盖以保证幂等。
## 参考文档
- Prefect 入门:https://docs.prefect.org.cn/v3/get-started
- API 驱动 ETL 示例:https://docs.prefect.org.cn/v3/examples/run-api-sourced-etl
- 密钥存储:https://docs.prefect.org.cn/v3/how-to-guides/configuration/store-secrets
+15
View File
@@ -0,0 +1,15 @@
"""Inkreach 外部系统 REST API 客户端集合。
子模块:
- api.riin : RIIN 分销平台(RiinDisClient)
- api.sds : SDS POD 平台(SdsMngClient)
- api._common : 共享 HTTP / 缓存 / 并发 / 批量 工具
这些客户端是纯 httpx + pydantic 实现,不依赖 Prefect。
Prefect 编排在项目根的 flows/ 目录。
"""
from api.riin import RiinDisClient
from api.sds import SdsMngClient
__all__ = ["RiinDisClient", "SdsMngClient"]
+15
View File
@@ -0,0 +1,15 @@
"""共享基础设施工具:HTTP 基类、TTL 缓存、并发、批量切分。"""
from api._common.batch import chunk, parse_id_text, split_lines
from api._common.cache import TTLCache
from api._common.concurrency import run_concurrent
from api._common.http import TokenHttpClient
__all__ = [
"TokenHttpClient",
"TTLCache",
"run_concurrent",
"parse_id_text",
"split_lines",
"chunk",
]
+36
View File
@@ -0,0 +1,36 @@
"""批量切分辅助:对应 TS SDK 里订单 OID/快递单号、SDS 订单关键词的批量拆分。"""
from __future__ import annotations
import re
_MULTI_SEP = re.compile(r"[,,\s\n\r]+")
def parse_id_text(text: str) -> list[str]:
"""把逗号/空白/换行分隔的字符串解析为去重的 ID 列表(RIIN 用)。"""
items = _MULTI_SEP.split(text or "")
seen: dict[str, None] = {}
for item in items:
item = item.strip()
if item and item not in seen:
seen[item] = None
return list(seen.keys())
def split_lines(text: str) -> list[str]:
"""把换行分隔的字符串解析为去重的列表(SDS 订单号 keywords 用)。"""
items = re.split(r"[\n\r]+", text or "")
seen: dict[str, None] = {}
for item in items:
item = item.strip()
if item and item not in seen:
seen[item] = None
return list(seen.keys())
def chunk(items: list[str], size: int) -> list[list[str]]:
"""把列表按 size 切成多批。"""
if not items:
return []
return [items[i : i + size] for i in range(0, len(items), size)]
+36
View File
@@ -0,0 +1,36 @@
"""简单的 TTL 内存缓存(纯 Python,无第三方依赖)。
对应 TS SDK 的 MemoryCache,用于主数据等就近缓存,生命周期跟随 client 实例。
注意:这是客户端层缓存,与 Prefect 的 task 缓存(cache_key_fn)是两回事。
"""
from __future__ import annotations
import time
from typing import Any, Generic, TypeVar
T = TypeVar("T")
_DEFAULT_TTL_SECONDS = 300
class TTLCache(Generic[T]):
def __init__(self, ttl_seconds: int | float = _DEFAULT_TTL_SECONDS) -> None:
self._ttl = ttl_seconds
self._store: dict[str, tuple[T, float]] = {}
def get(self, key: str) -> T | None:
item = self._store.get(key)
if item is None:
return None
value, expires_at = item
if time.monotonic() > expires_at:
self._store.pop(key, None)
return None
return value
def set(self, key: str, value: T) -> None:
self._store[key] = (value, time.monotonic() + self._ttl)
def clear(self) -> None:
self._store.clear()
+22
View File
@@ -0,0 +1,22 @@
"""并发执行辅助:线程池 + 信号量,对应 TS SDK 的 ConcurrentLimiter(默认 10)。"""
from __future__ import annotations
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from typing import Any, TypeVar
T = TypeVar("T")
_DEFAULT_CONCURRENCY = 10
def run_concurrent(
funcs: list[Callable[[], T]],
limit: int = _DEFAULT_CONCURRENCY,
) -> list[T]:
"""并发执行一批无参 callable(每个返回一个结果),保持输入顺序返回结果。"""
if not funcs:
return []
with ThreadPoolExecutor(max_workers=max(1, min(limit, len(funcs)))) as pool:
return list(pool.map(lambda fn: fn(), funcs))
+98
View File
@@ -0,0 +1,98 @@
"""基础 HTTP 客户端:token 注入 + 401 自动重登。
两个外部系统(RIIN / SDS)共享这套行为,仅以下差异由子类配置:
- base_url
- token 写入的请求头名(RIIN: Authorization,SDS: access-token)
- 固定请求头(RIIN 带 X-Time-Zone / X-Requested-With,SDS 无)
- 登录实现(子类实现 _login)
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
import httpx
class TokenHttpClient(ABC):
"""带 token 鉴权的 HTTP 客户端基类(组合式,非强制耦合)。"""
def __init__(
self,
base_url: str,
*,
token_header: str,
fixed_headers: dict[str, str] | None = None,
login_markers: tuple[str, ...] = (),
timeout: float = 30.0,
) -> None:
self._token: str | None = None
self._token_header = token_header
self._login_markers = login_markers
self._http = httpx.Client(
base_url=base_url,
headers=fixed_headers or {},
timeout=timeout,
)
@abstractmethod
def _login(self) -> str:
"""执行登录并返回新 token。子类实现具体认证逻辑。"""
# ----- token 生命周期 -----
def _ensure_token(self) -> str:
if not self._token:
self._token = self._login()
return self._token
def _clear_token(self) -> None:
self._token = None
def _is_login_call(self, url: str) -> bool:
return any(marker in url for marker in self._login_markers)
# ----- 请求 -----
def _send(
self,
method: str,
url: str,
*,
params: dict[str, Any] | None = None,
body: dict[str, Any] | None = None,
_retry: bool = True,
) -> Any:
headers: dict[str, str] = {}
if not self._is_login_call(url):
headers[self._token_header] = self._ensure_token()
response = self._http.request(method, url, params=params, json=body, headers=headers)
if response.status_code == 401 and _retry and not self._is_login_call(url):
self._clear_token()
return self._send(method, url, params=params, body=body, _retry=False)
response.raise_for_status()
return response.json()
def get(self, url: str, params: dict[str, Any] | None = None) -> Any:
return self._send("GET", url, params=params)
def post(
self,
url: str,
body: dict[str, Any] | None = None,
params: dict[str, Any] | None = None,
) -> Any:
return self._send("POST", url, body=body, params=params)
def close(self) -> None:
self._http.close()
def __enter__(self) -> "TokenHttpClient":
return self
def __exit__(self, *exc: Any) -> None:
self.close()
+6
View File
@@ -0,0 +1,6 @@
"""RIIN 分销平台 REST API 客户端。"""
from api.riin.client import RiinDisClient
from api.riin.models import OrderPageRequest
__all__ = ["RiinDisClient", "OrderPageRequest"]
+329
View File
@@ -0,0 +1,329 @@
"""RIIN 分销平台 REST API 客户端(移植自 @inkreach/riin-dis)。
Base URL: https://tshirt.riin.com
认证: username + MD5(password) -> Bearer token(Authorization 头),401 自动重登。
"""
from __future__ import annotations
import hashlib
from typing import Any
from pydantic import BaseModel, ConfigDict
from pydantic.alias_generators import to_camel
from api._common import TTLCache, chunk, parse_id_text, run_concurrent
from api._common.http import TokenHttpClient
from api.riin.models import (
Article,
Color,
CustomerListResponse,
CustomerShopListResponse,
ExpressInfo,
FactoryInfoListResponse,
LoginResult,
OrderGoodsResponse,
OrderListCountResponse,
OrderLogResponse,
OrderPageRequest,
OrderPageResponse,
Size,
ThirdInterfaceCustomer,
)
BASE_URL = "https://tshirt.riin.com"
BATCH_SIZE = 500
class _ListResponse(BaseModel):
"""通用列表响应壳(辅助主数据用)。"""
model_config = ConfigDict(extra="allow", populate_by_name=True, alias_generator=to_camel)
successful: bool | None = None
message: str | None = None
error_code: str | None = None
data: list[Any] = []
class _PageResponse(BaseModel):
model_config = ConfigDict(extra="allow", populate_by_name=True, alias_generator=to_camel)
successful: bool | None = None
message: str | None = None
error_code: str | None = None
data: dict[str, Any] = {}
class RiinDisClient(TokenHttpClient):
"""RIIN 分销系统客户端:认证 / 订单 / 店铺 / 辅助主数据。"""
def __init__(
self,
username: str,
password: str,
*,
cache_ttl_seconds: int | float = 300,
concurrency: int = 10,
) -> None:
super().__init__(
BASE_URL,
token_header="Authorization",
fixed_headers={"X-Time-Zone": "UTC+8", "X-Requested-With": "XMLHttpRequest"},
login_markers=("auth/api/auth/login",),
)
self._username = username
self._password = password
self._cache: TTLCache[Any] = TTLCache(cache_ttl_seconds)
self._concurrency = concurrency
# ----- 认证 -----
def _login(self) -> str:
md5_password = hashlib.md5(self._password.encode()).hexdigest()
data = self.post("auth/api/auth/login", body={"username": self._username, "password": md5_password})
result = LoginResult.model_validate(data["data"])
self._token = result.token
return self._token
# ----- 订单 -----
def _build_order_page_body(self, params: OrderPageRequest) -> dict[str, Any]:
has_oid_or_courier = bool(params.platform_oid_list or params.courier_number_list)
body: dict[str, Any] = {
"pageIndex": params.page_index if params.page_index is not None else 1,
"pageSize": params.page_size if params.page_size is not None else 20,
"shopCodeList": params.shop_code_list or [],
"orderStatus": params.order_status if params.order_status is not None else None,
"dayRangeGroup": params.day_range_group if params.day_range_group is not None else None,
"hasRefund": params.has_refund if params.has_refund is not None else False,
}
if not has_oid_or_courier:
body["timeType"] = params.time_type if params.time_type is not None else 3
if params.start_time:
body["startTime"] = params.start_time
if params.end_time:
body["endTime"] = params.end_time
elif params.time_type is not None:
body["timeType"] = params.time_type
if params.platform_oid_list:
body["platformOidList"] = parse_id_text(params.platform_oid_list)
if params.courier_number_list:
body["courierNumberList"] = parse_id_text(params.courier_number_list)
if params.sub_id_list:
body["subIdList"] = params.sub_id_list
if params.goods_title:
body["goodsTitle"] = params.goods_title
if params.mapping_sku_code:
body["mappingSkuCode"] = params.mapping_sku_code
if params.platform_type is not None:
body["platformType"] = params.platform_type
if params.order_source is not None:
body["orderSource"] = params.order_source
if params.factory_ids:
body["factoryIds"] = params.factory_ids
if params.express_source is not None:
body["expressSource"] = params.express_source
if params.delivery_courier:
body["deliveryCourier"] = params.delivery_courier
if params.receiver_country:
body["receiverCountry"] = params.receiver_country
if params.region:
body["region"] = params.region
if params.receiver_province:
body["receiverProvince"] = params.receiver_province
if params.craft_type is not None:
body["craftType"] = params.craft_type
if params.style_code_list:
body["styleCodeList"] = params.style_code_list
if params.color_code_list:
body["colorCodeList"] = params.color_code_list
if params.size_code_list:
body["sizeCodeList"] = params.size_code_list
if params.seller_remark:
body["sellerRemark"] = params.seller_remark
if params.urgent_order is not None:
body["urgentOrder"] = params.urgent_order
if params.third_interface_customer_id:
body["thirdInterfaceCustomerId"] = params.third_interface_customer_id
if params.platform_status_list:
body["platformStatusList"] = params.platform_status_list
if params.is_third_flag is not None:
body["isThirdFlag"] = params.is_third_flag
if params.push_failed is not None:
body["pushFailed"] = params.push_failed
if params.assigned_fail_reason:
body["assignedFailReason"] = params.assigned_fail_reason
if params.push_fail_reason:
body["pushFailReason"] = params.push_fail_reason
if params.factory_remark_param:
body["factoryRemarkParam"] = params.factory_remark_param
if params.distributor_remark_param:
body["distributorRemarkParam"] = params.distributor_remark_param
return body
def _get_single_page(self, params: OrderPageRequest) -> OrderPageResponse:
body = self._build_order_page_body(
params.model_copy(update={"page_index": params.page_index or 1, "page_size": params.page_size or 20})
)
data = self.post("distribution/order/page", body=body)
return OrderPageResponse.model_validate(data)
def _get_all_pages(self, params: OrderPageRequest) -> OrderPageResponse:
page_size = params.page_size or 200
first = self._get_single_page(params.model_copy(update={"page_index": 1, "page_size": page_size}))
total_count = first.data.total
total_pages = -(-total_count // page_size) # ceil
if total_pages <= 1:
return first
remaining_indices = list(range(2, total_pages + 1))
def fetch(i: int) -> OrderPageResponse:
return self._get_single_page(params.model_copy(update={"page_index": i, "page_size": page_size}))
remaining = run_concurrent([lambda i=i: fetch(i) for i in remaining_indices], limit=self._concurrency)
records = list(first.data.records)
for r in remaining:
records.extend(r.data.records)
return OrderPageResponse(
successful=True,
data={"records": records, "total": total_count},
)
def get_orders(self, params: OrderPageRequest | None = None) -> OrderPageResponse:
"""获取订单。支持三种模式:limit(取首页 N 条)/ all(全量分页)/ 单页。"""
params = params or OrderPageRequest()
needs_batching = (
(params.platform_oid_list and len(parse_id_text(params.platform_oid_list)) > BATCH_SIZE)
or (params.courier_number_list and len(parse_id_text(params.courier_number_list)) > BATCH_SIZE)
)
if needs_batching:
return self._get_orders_batched(params)
if params.limit:
return self._get_single_page(params.model_copy(update={"page_index": 1, "page_size": params.limit}))
if params.all:
return self._get_all_pages(params)
return self._get_single_page(params)
def _get_orders_batched(self, params: OrderPageRequest) -> OrderPageResponse:
oid_batches = chunk(parse_id_text(params.platform_oid_list), BATCH_SIZE) if params.platform_oid_list else [None]
courier_batches = (
chunk(parse_id_text(params.courier_number_list), BATCH_SIZE) if params.courier_number_list else [None]
)
def fetch(oid_batch: list[str] | None, courier_batch: list[str] | None) -> OrderPageResponse:
update: dict[str, Any] = {}
if oid_batch is not None:
update["platform_oid_list"] = ",".join(oid_batch)
if courier_batch is not None:
update["courier_number_list"] = ",".join(courier_batch)
batch_params = params.model_copy(update=update)
if params.limit:
return self._get_single_page(
batch_params.model_copy(update={"page_index": 1, "page_size": params.limit})
)
return self._get_all_pages(batch_params)
calls = [
(lambda o=o, c=c: fetch(o, c)) for o in oid_batches for c in courier_batches
]
results = run_concurrent(calls, limit=self._concurrency)
records: list[Any] = []
total = 0
for r in results:
records.extend(r.data.records)
total += r.data.total
if params.limit:
records = records[: params.limit]
return OrderPageResponse(successful=True, data={"records": records, "total": total})
def get_order_goods(self, order_id: str) -> OrderGoodsResponse:
data = self.post("distribution/order/goods", body={"orderId": order_id})
return OrderGoodsResponse.model_validate(data)
def get_order_count(
self, params: OrderPageRequest | None = None
) -> OrderListCountResponse:
params = params or OrderPageRequest()
cache_key = f"orderCount:{params.model_dump_json(exclude_none=True)}"
cached = self._cache.get(cache_key)
if cached is not None:
return cached
body = self._build_order_page_body(params)
body.pop("pageIndex", None)
body.pop("pageSize", None)
data = self.post("distribution/order/list/count", body=body)
result = OrderListCountResponse.model_validate(data)
self._cache.set(cache_key, result)
return result
def get_order_logs(self, order_id: str, platform_oid: str) -> OrderLogResponse:
data = self.post("distribution/order/logs", body={"orderId": order_id, "platformOid": platform_oid})
return OrderLogResponse.model_validate(data)
# ----- 店铺 / 工厂 / 客户 -----
def get_customer_shop_list(self) -> CustomerShopListResponse:
cache_key = "shops:customerShopList"
cached = self._cache.get(cache_key)
if cached is not None:
return cached
data = self.post("distribution/order/customer/shop/list", body={})
result = CustomerShopListResponse.model_validate(data)
self._cache.set(cache_key, result)
return result
def get_factory_list(self) -> FactoryInfoListResponse:
cache_key = "shops:factoryList"
cached = self._cache.get(cache_key)
if cached is not None:
return cached
data = self.get("distribution/tenant/getFactoryInfoList")
result = FactoryInfoListResponse.model_validate(data)
self._cache.set(cache_key, result)
return result
def get_customer_list(self) -> CustomerListResponse:
cache_key = "shops:customerList"
cached = self._cache.get(cache_key)
if cached is not None:
return cached
data = self.post("distribution/customer/listTenantNoDisable", body={})
result = CustomerListResponse.model_validate(data)
self._cache.set(cache_key, result)
return result
# ----- 辅助主数据 -----
def get_articles(self, params: dict[str, Any] | None = None) -> list[Article]:
data = self.post("distribution/articleManagement/list", body=params or {})
resp = _ListResponse.model_validate(data)
return [Article.model_validate(item) for item in resp.data]
def get_sizes(self, params: dict[str, Any] | None = None) -> list[Size]:
data = self.post("distribution/sizeManagement/size/list", body=params or {})
resp = _ListResponse.model_validate(data)
return [Size.model_validate(item) for item in resp.data]
def get_colors(self, params: dict[str, Any] | None = None) -> list[Color]:
data = self.post("distribution/colorManagement/list", body=params or {})
resp = _ListResponse.model_validate(data)
return [Color.model_validate(item) for item in resp.data]
def get_third_party_customers(
self, params: dict[str, Any] | None = None
) -> list[ThirdInterfaceCustomer]:
data = self.post("distribution/thirdInterfaceCustomer/page", body=params or {})
resp = _PageResponse.model_validate(data)
records = resp.data.get("records", []) if isinstance(resp.data, dict) else []
return [ThirdInterfaceCustomer.model_validate(item) for item in records]
def get_express_info(self) -> list[ExpressInfo]:
data = self.post("distribution/order/tenant/delivery/express", body={})
resp = _ListResponse.model_validate(data)
return [ExpressInfo.model_validate(item) for item in resp.data]
+409
View File
@@ -0,0 +1,409 @@
"""RIIN 分销平台 —— 数据模型(pydantic v2)。
忠实还原 @inkreach/riin-dis 的 TypeScript 类型,extra='allow' 兼容未知字段。
"""
from __future__ import annotations
from enum import IntEnum
from typing import Any
from pydantic import BaseModel, ConfigDict
from pydantic.alias_generators import to_camel
class PlatformType(IntEnum):
TIKTOK = 11
TEMU = 13
SHEIN = 17
WALMART = 19
class ShopType(IntEnum):
TIKTOK = 11
TEMU = 13
SHEIN = 17
WALMART = 19
class OrderStatusValue(IntEnum):
NOT_PUSHED = -4
PENDING_REVIEW = -5
ASSIGN_FAILED = -6
UNDER_REVIEW = -3
PENDING_PUSH = 1
UNDER_REVIEW_DISTRIBUTOR = 3
PUSHED = 4
IN_PRODUCTION = 5
SHIPPED = 12
COMPLETED = 13
CANCELLED = 14
CLOSED = 15
DAY_RANGE_3 = 101
DAY_RANGE_3_7 = 102
DAY_RANGE_7_15 = 103
DAY_RANGE_15_PLUS = 104
class ExpressSource(IntEnum):
FACTORY = 1
PLATFORM_LABEL = 2
SELF_LABEL = 3
class TimeType(IntEnum):
ORDER_CREATE = 1
PUSH_FACTORY = 5
MERCHANT_PUSH = 3
SHIP = 4
class _RiinModel(BaseModel):
model_config = ConfigDict(
extra="allow",
populate_by_name=True,
alias_generator=to_camel,
)
# ----- 认证 -----
class LoginResult(_RiinModel):
id: str | None = None
tenant_id: str | None = None
sub_id: str | None = None
tenant_name: str | None = None
username: str | None = None
nickname: str | None = None
user_type: str | None = None
token: str
token_flag: str | None = None
invite_code: str | None = None
main_account: bool | None = None
factory: bool | None = None
distribution: bool | None = None
sub_distribution: bool | None = None
# ----- 订单 -----
class ExpressItem(_RiinModel):
express_name: str | None = None
ship_logistics_type: str | None = None
class Order(_RiinModel):
id: str
tenant_id: str | None = None
sub_id: str | None = None
sub_name: str | None = None
source_platform_oid: str | None = None
platform_oid: str | None = None
platform_order_name: str | None = None
order_status: int | None = None
order_type: int | None = None
goods_total_qty: int | None = None
platform_order_status: str | None = None
platform_refund_status: str | None = None
platform_shop_code: str | None = None
platform_shop_id: str | None = None
platform_shop_name: str | None = None
platform_type: int | None = None
sub_platform_type: int | None = None
seller_flag: str | None = None
factory_remark: str | None = None
distributor_remark: str | None = None
seller_remark: str | None = None
buyer_remark: str | None = None
delivery_courier: str | None = None
express_code: str | None = None
express_fee: float | None = None
express_source: int | None = None
courier_number: str | None = None
consignee_name: str | None = None
phone: str | None = None
email: str | None = None
address: str | None = None
receiver_country: str | None = None
receiver_province: str | None = None
receiver_city: str | None = None
receiver_district: str | None = None
receiver_town: str | None = None
post_code: str | None = None
currency: str | None = None
currency_type: str | None = None
total_fee: float | None = None
payment: float | None = None
received_payment: float | None = None
total_price: float | None = None
total_sell_price: float | None = None
post_fee: float | None = None
order_create_time: str | None = None
pre_shipping_time: str | None = None
shipping_time: str | None = None
order_pay_time: str | None = None
order_time: str | None = None
finish_time: str | None = None
cancel_time: str | None = None
audit_time: str | None = None
push_time: str | None = None
distributor_push_time: str | None = None
distributor_push_target_name: str | None = None
except_last_shipping_time: str | None = None
push_msg: str | None = None
pushed_failed: str | None = None
assigned_fail_reason: str | None = None
urgent_order: int | None = None
is_third_flag: bool | None = None
is_custom_product: bool | None = None
express_list: list[ExpressItem] = []
class OrderPageData(_RiinModel):
records: list[Order] = []
total: int = 0
class OrderPageResponse(_RiinModel):
successful: bool | None = None
message: str | None = None
error_code: str | None = None
data: OrderPageData = OrderPageData()
class PrintingImage(_RiinModel):
id: str | None = None
type: int | None = None
url: str | None = None
name: str | None = None
code: str | None = None
class OrderGoodsImageObjs(_RiinModel):
sku_image: str | None = None
printing_image: str | None = None
printing_images: list[PrintingImage] = []
sku_images: list[str] | None = None
print_urls: list[str] = []
sku_urls: list[str] = []
class OrderGoods(_RiinModel):
id: str
order_id: str | None = None
platform_oid: str | None = None
platform_spu_id: str | None = None
platform_sku_id: str | None = None
title: str | None = None
specification: str | None = None
goods_status: str | None = None
refund_status: str | None = None
price: float | None = None
sell_price: float | None = None
num: int | None = None
total_price: float | None = None
total_sell_price: float | None = None
sku_image: str | None = None
mapping_sku_code: str | None = None
factory_mapping_sku_code: str | None = None
style_code: str | None = None
image_code: str | None = None
size_code: str | None = None
color_code: str | None = None
style_name: str | None = None
image_name: str | None = None
size_name: str | None = None
color_name: str | None = None
product_code: str | None = None
craft_type: int | None = None
print_position: str | None = None
image_objs: Any | None = None
class OrderGoodsResponse(_RiinModel):
successful: bool | None = None
message: str | None = None
error_code: str | None = None
data: list[OrderGoods] = []
class OrderListCount(_RiinModel):
assigned_suceessed: int = 0
factory_auditing: int = 0
in_producing: int = 0
shipped: int = 0
closed: int = 0
refunding: int = 0
refunded: int = 0
rejecting: int = 0
rejected_to_shop: int = 0
day_range_group1: int = 0
day_range_group2: int = 0
day_range_group3: int = 0
day_range_group4: int = 0
class OrderListCountResponse(_RiinModel):
successful: bool | None = None
message: str | None = None
error_code: str | None = None
data: OrderListCount = OrderListCount()
class OrderLog(_RiinModel):
id: str | None = None
order_id: str | None = None
platform_oid: str | None = None
operate_type: str | None = None
operate_msg: str | None = None
operator_name: str | None = None
operate_time: str | None = None
class OrderLogResponse(_RiinModel):
successful: bool | None = None
message: str | None = None
error_code: str | None = None
data: list[OrderLog] = []
# ----- 店铺 / 工厂 / 客户 -----
class CustomerShop(_RiinModel):
tenant_id: str | None = None
tenant_name: str | None = None
shop_id: str | None = None
shop_code: str | None = None
shop_name: str | None = None
shop_type: int | None = None
sub_id: str | None = None
class CustomerShopListResponse(_RiinModel):
successful: bool | None = None
message: str | None = None
error_code: str | None = None
data: list[CustomerShop] = []
class FactoryInfo(_RiinModel):
factory_id: str | None = None
phone: str | None = None
contact: str | None = None
address: str | None = None
name: str | None = None
logo_url: str | None = None
class FactoryInfoListResponse(_RiinModel):
successful: bool | None = None
message: str | None = None
error_code: str | None = None
data: list[FactoryInfo] = []
class Customer(_RiinModel):
id: str | None = None
tenant_id: str | None = None
customer_tenant_id: str | None = None
customer_name: str | None = None
username: str | None = None
contacts: str | None = None
mobile: str | None = None
settlement_currency: int | None = None
disable: int | None = None
class CustomerListResponse(_RiinModel):
successful: bool | None = None
message: str | None = None
error_code: str | None = None
data: list[Customer] = []
# ----- 辅助主数据 -----
class Article(_RiinModel):
id: str | None = None
code: str | None = None
name: str | None = None
class Size(_RiinModel):
id: str | None = None
code: str | None = None
name: str | None = None
class Color(_RiinModel):
id: str | None = None
code: str | None = None
name: str | None = None
class ThirdInterfaceCustomer(_RiinModel):
id: str | None = None
name: str | None = None
class ExpressInfo(_RiinModel):
id: str | None = None
name: str | None = None
code: str | None = None
class PageData(_RiinModel):
records: list[Any] = []
total: int = 0
# ----- 查询参数(请求 schema)-----
class OrderPageRequest(BaseModel):
"""RIIN 订单分页查询参数。timeType/startTime/endTime 等可选。"""
model_config = ConfigDict(extra="allow")
time_type: int | None = None
start_time: str | None = None
end_time: str | None = None
platform_oid_list: str | None = None
shop_code_list: list[str] | None = None
courier_number_list: str | None = None
sub_id_list: list[str] | None = None
goods_title: str | None = None
mapping_sku_code: str | None = None
platform_type: int | None = None
order_source: int | None = None
factory_ids: list[str] | None = None
express_source: int | None = None
delivery_courier: str | None = None
receiver_country: str | None = None
region: str | None = None
receiver_province: str | None = None
craft_type: int | None = None
style_code_list: list[str] | None = None
color_code_list: list[str] | None = None
size_code_list: list[str] | None = None
seller_remark: str | None = None
urgent_order: int | None = None
third_interface_customer_id: str | None = None
platform_status_list: list[str] | None = None
is_third_flag: bool | None = None
page_index: int | None = None
page_size: int | None = None
order_status: int | None = None
day_range_group: int | None = None
has_refund: bool | None = None
push_failed: bool | None = None
assigned_fail_reason: str | None = None
push_fail_reason: str | None = None
factory_remark_param: str | None = None
distributor_remark_param: str | None = None
limit: int | None = None
all: bool | None = None
+6
View File
@@ -0,0 +1,6 @@
"""SDS POD 平台 REST API 客户端。"""
from api.sds.client import SdsMngClient
from api.sds.models import OrderPageRequest
__all__ = ["SdsMngClient", "OrderPageRequest"]
+354
View File
@@ -0,0 +1,354 @@
"""SDS POD 平台 REST API 客户端(移植自 @inkreach/sds-mng)。
Base URL: https://pod-api.sdspod.com
认证: username + password + 公司名(印美达) -> access-token 头,401 自动重登。
"""
from __future__ import annotations
import time
from typing import Any
from api._common import TTLCache, chunk, run_concurrent, split_lines
from api._common.http import TokenHttpClient
from api.sds.models import (
FiltratePanelResponse,
IssuingBayAreaResponse,
LogisticsItem,
LogisticsPageRequest,
LogisticsPageResponse,
LoginResult,
OrderPageRequest,
OrderPageResponse,
OrderQueryParams,
OrderStatistics,
OrderStatus,
ProductListRequest,
ProductListResponse,
)
BASE_URL = "https://pod-api.sdspod.com"
COMPANY_NO = "印美达"
KEYWORDS_BATCH_SIZE = 200
FILTERS_FIELDS = (
"factoryList,merchantList,logisticsList,earlyWarningList,"
"issuingBayAreaList,logisticsSourceList,extendStatusList"
)
EXTRA_INFO = (
"eyJpcCI6IjM4LjcxLjEyNy4xODkiLCJsb2NhdGlvbiI6eyJsYXQiOjM4Ljg4MzMsImxuZyI6LTc3fSwiYWRfaW5mbyI6"
"eyJuYXRpb24iOiLnrKw0OSIsInByb3ZpbmNlIjoiIiwiY2l0eSI6IiIsImRpc3RyaWN0IjoiIiwiYWRjb2RlIjotMSwi"
"bmF0aW9uX2NvZGUiOjg0MH0sImZpbmdlcnByaW50SWQiOiI3YzdjOWUwNTFhZTY0OTM5MTMwZjk5YjQ2OGJjYWFhNSIs"
"InVzZXJBZ2VudCI6Ik1vemlsbGEvNS4wIChXaW5kb3dzIE5UIDEwLjA7IFdpbjY0OyB4NjQpIEFwcGxlV2ViS2l0LzUz"
"Ny4zNiAoS0hUTUwsIGxpa2UgR2Vja28pIENocm9tZS8xMzguMC4wLjAgU2FmYXJpLzUzNy4zNiIsImhyZWYiOiJodHRw"
"czovL21hbmFnZS5zZHNwb2QuY29tL3VzZXIvbG9naW4ifQ=="
)
class SdsMngClient(TokenHttpClient):
"""SDS POD 系统客户端:认证 / 订单 / 产品 / 物流。"""
def __init__(
self,
username: str,
password: str,
*,
cache_ttl_seconds: int | float = 300,
concurrency: int = 10,
) -> None:
super().__init__(
BASE_URL,
token_header="access-token",
fixed_headers=None,
login_markers=("/pod/auth/login",),
)
self._username = username
self._password = password
self._cache: TTLCache[Any] = TTLCache(cache_ttl_seconds)
self._concurrency = concurrency
# ----- 认证 -----
def _login(self) -> str:
data = self.post(
f"/pod/auth/login?t={int(time.time() * 1000)}",
body={
"no": COMPANY_NO,
"password": self._password,
"username": self._username,
"extraInfo": EXTRA_INFO,
},
)
result = LoginResult.model_validate(data["data"])
self._token = result.access_token
return self._token
# ----- 订单 -----
@staticmethod
def _build_order_params(params: OrderPageRequest) -> dict[str, Any]:
query: dict[str, Any] = {"productionType": 2, "t": int(time.time() * 1000)}
if params.size is not None:
query["size"] = params.size
if params.page is not None:
query["page"] = params.page
if params.keywords is not None:
query["keywords"] = params.keywords
if params.created_date_begin is not None:
query["createdDateBegin"] = params.created_date_begin
if params.created_date_end is not None:
query["createdDateEnd"] = params.created_date_end
if params.pay_date_begin is not None:
query["payDateBegin"] = params.pay_date_begin
if params.pay_date_end is not None:
query["payDateEnd"] = params.pay_date_end
if params.confirm_date_begin is not None:
query["confirmDateBegin"] = params.confirm_date_begin
if params.confirm_date_end is not None:
query["confirmDateEnd"] = params.confirm_date_end
if params.finish_date_begin is not None:
query["finishDateBegin"] = params.finish_date_begin
if params.finish_date_end is not None:
query["finishDateEnd"] = params.finish_date_end
if params.order_status_code is not None:
query["orderStatusCode"] = params.order_status_code
if params.product_keywords is not None:
query["productKeywords"] = params.product_keywords
if params.order_time_range_type is not None:
query["orderTimeRangeType"] = params.order_time_range_type
if params.factory_ids is not None:
query["factoryIds"] = params.factory_ids
if params.merchant_ids is not None:
query["merchantIds"] = params.merchant_ids
if params.logistics_ids is not None:
query["logisticsIds"] = params.logistics_ids
if params.issuing_bay_area_id is not None:
query["issuingBayAreaId"] = params.issuing_bay_area_id
if params.extend_status is not None:
query["extendStatus"] = params.extend_status
if params.early_warning_type is not None:
query["earlyWarningType"] = params.early_warning_type
if params.item_num_type is not None:
query["itemNumType"] = params.item_num_type
return query
@staticmethod
def _build_query_params(params: OrderQueryParams | None) -> dict[str, Any]:
params = params or OrderQueryParams()
query: dict[str, Any] = {"t": int(time.time() * 1000)}
for py_name, api_name in [
("created_date_begin", "createdDateBegin"),
("created_date_end", "createdDateEnd"),
("pay_date_begin", "payDateBegin"),
("pay_date_end", "payDateEnd"),
("confirm_date_begin", "confirmDateBegin"),
("confirm_date_end", "confirmDateEnd"),
("finish_date_begin", "finishDateBegin"),
("finish_date_end", "finishDateEnd"),
("order_time_range_type", "orderTimeRangeType"),
("product_keywords", "productKeywords"),
("factory_ids", "factoryIds"),
("merchant_ids", "merchantIds"),
("logistics_ids", "logisticsIds"),
("issuing_bay_area_id", "issuingBayAreaId"),
("extend_status", "extendStatus"),
("early_warning_type", "earlyWarningType"),
("item_num_type", "itemNumType"),
]:
value = getattr(params, py_name, None)
if value:
query[api_name] = value
return query
def _get_single_page(self, params: OrderPageRequest) -> OrderPageResponse:
query = self._build_order_params(
params.model_copy(update={"size": params.size or 50, "page": params.page or 1})
)
data = self.get("/pod/orders/page", params=query)
return OrderPageResponse.model_validate(data)
def _get_all_pages(self, params: OrderPageRequest) -> OrderPageResponse:
page_size = params.size or 200
first = self._get_single_page(params.model_copy(update={"size": page_size, "page": 1}))
total_pages = -(-first.total_count // page_size)
if total_pages <= 1:
return first
def fetch(i: int) -> OrderPageResponse:
return self._get_single_page(params.model_copy(update={"size": page_size, "page": i}))
remaining = run_concurrent(
[lambda i=i: fetch(i) for i in range(2, total_pages + 1)], limit=self._concurrency
)
merged = list(first.records)
for r in remaining:
merged.extend(r.records)
return OrderPageResponse(total_count=first.total_count, records=merged)
def get_orders(self, params: OrderPageRequest | None = None) -> OrderPageResponse:
"""获取订单。keywords>200 自动分批;limit 取首页 N 条;all 全量分页。"""
params = params or OrderPageRequest()
if params.keywords:
return self._get_orders_with_keywords(params)
if params.limit:
return self._get_single_page(params.model_copy(update={"size": params.limit, "page": 1}))
if params.all:
return self._get_all_pages(params)
return self._get_single_page(params)
def _get_orders_with_keywords(self, params: OrderPageRequest) -> OrderPageResponse:
batches = chunk(split_lines(params.keywords or ""), KEYWORDS_BATCH_SIZE) or [[]]
def fetch(batch: list[str]) -> OrderPageResponse:
batch_params = params.model_copy(update={"keywords": "\n".join(batch)})
if params.limit:
return self._get_single_page(
batch_params.model_copy(update={"size": params.limit, "page": 1})
)
return self._get_all_pages(batch_params)
results = run_concurrent([lambda b=b: fetch(b) for b in batches], limit=self._concurrency)
merged: list[Any] = []
total = 0
for r in results:
merged.extend(r.records)
total += r.total_count
if params.limit:
merged = merged[: params.limit]
return OrderPageResponse(total_count=total, list=merged)
def get_filters(self, params: OrderQueryParams | None = None) -> FiltratePanelResponse:
cache_key = f"filters:{(params.model_dump_json(exclude_none=True)) if params else '{}'}"
cached = self._cache.get(cache_key)
if cached is not None:
return cached
query = {"fields": FILTERS_FIELDS, **self._build_query_params(params)}
data = self.get("/pod/orders/filtratePanel", params=query)
result = FiltratePanelResponse.model_validate(data)
self._cache.set(cache_key, result)
return result
def get_status(self, params: OrderQueryParams | None = None) -> list[OrderStatus]:
cache_key = f"status:{(params.model_dump_json(exclude_none=True)) if params else '{}'}"
cached = self._cache.get(cache_key)
if cached is not None:
return cached
query = {"productionType": 2, **self._build_query_params(params)}
data = self.get("/pod/orders/statistics", params=query)
result = [OrderStatus.model_validate(item) for item in data]
self._cache.set(cache_key, result)
return result
def get_statistics(
self, params: OrderQueryParams | None = None, production_type: int = 2
) -> list[OrderStatistics]:
params = params or OrderQueryParams()
cache_key = f"statistics:{production_type}:{params.model_dump_json(exclude_none=True)}"
cached = self._cache.get(cache_key)
if cached is not None:
return cached
query = {
"fields": FILTERS_FIELDS,
"productionType": production_type,
**self._build_query_params(params),
}
data = self.get("/pod/orders/statistics", params=query)
result = [OrderStatistics.model_validate(item) for item in data]
self._cache.set(cache_key, result)
return result
# ----- 产品 -----
def get_products(self, params: ProductListRequest | None = None) -> ProductListResponse:
params = params or ProductListRequest()
if params.limit:
return self._get_products_page(params.model_copy(update={"size": params.limit, "page": 1}))
if params.all:
return self._get_all_products(params)
return self._get_products_page(params)
def _get_products_page(self, params: ProductListRequest) -> ProductListResponse:
query = {
"keyword": params.keyword or "",
"page": params.page or 1,
"size": params.size or 50,
"publicStatus": params.public_status if params.public_status is not None else 1,
"t": int(time.time() * 1000),
}
data = self.get("/pod/product/self", params=query)
return ProductListResponse.model_validate(data)
def _get_all_products(self, params: ProductListRequest) -> ProductListResponse:
page_size = params.size or 200
first = self._get_products_page(params.model_copy(update={"size": page_size, "page": 1}))
total_pages = -(-first.total_count // page_size)
if total_pages <= 1:
return first
remaining = run_concurrent(
[
lambda i=i: self._get_products_page(params.model_copy(update={"size": page_size, "page": i}))
for i in range(2, total_pages + 1)
],
limit=self._concurrency,
)
merged = list(first.records)
for r in remaining:
merged.extend(r.records)
return ProductListResponse(total_count=first.total_count, records=merged)
# ----- 物流 -----
def get_logistics_areas(self) -> IssuingBayAreaResponse:
cache_key = "logistics:areas"
cached = self._cache.get(cache_key)
if cached is not None:
return cached
data = self.get("/pod/issuing_bay_areas", params={"t": int(time.time() * 1000)})
result = IssuingBayAreaResponse.model_validate(data)
self._cache.set(cache_key, result)
return result
def get_logistics_page(self, params: LogisticsPageRequest) -> LogisticsPageResponse:
if params.limit:
return self._get_logistics_single(
params.model_copy(update={"size": params.limit, "page": 1})
)
if params.all:
return self._get_logistics_all(params)
return self._get_logistics_single(params)
def _get_logistics_single(self, params: LogisticsPageRequest) -> LogisticsPageResponse:
query = {
"keyword": params.keyword or "",
"deliveryMethod": params.delivery_method or "FBM",
"issuingBayAreaId": params.issuing_bay_area_id,
"page": params.page or 1,
"size": params.size or 50,
"t": int(time.time() * 1000),
}
data = self.get("/pod/logistics/page", params=query)
return LogisticsPageResponse.model_validate(data)
def _get_logistics_all(self, params: LogisticsPageRequest) -> LogisticsPageResponse:
page_size = params.size or 200
first = self._get_logistics_single(
params.model_copy(update={"size": page_size, "page": 1})
)
total_count = first.total_count or first.total_count_alt or 0
total_pages = -(-total_count // page_size)
if total_pages <= 1:
return first
remaining = run_concurrent(
[
lambda i=i: self._get_logistics_single(params.model_copy(update={"size": page_size, "page": i}))
for i in range(2, total_pages + 1)
],
limit=self._concurrency,
)
merged: list[LogisticsItem] = list(first.items)
for r in remaining:
merged.extend(r.items)
return LogisticsPageResponse(
items=merged, page=1, size=len(merged), total_count=total_count
)
+364
View File
@@ -0,0 +1,364 @@
"""SDS POD 平台 —— 数据模型(pydantic v2)。
移植自 @inkreach/sds-mng。订单域为 camelCase;物流域混用 snake_case,
对 snake_case 字段显式标注 alias(alias 优先于 alias_generator)。
所有模型 extra='allow',不丢字段。
"""
from __future__ import annotations
from typing import Any
from pydantic import BaseModel, ConfigDict, Field
from pydantic.alias_generators import to_camel
class _SdsModel(BaseModel):
model_config = ConfigDict(
extra="allow",
populate_by_name=True,
alias_generator=to_camel,
)
# ----- 认证 -----
class LoginResult(_SdsModel):
access_token: str
# ----- 订单域(camelCase)-----
class MerchantStore(_SdsModel):
id: int | None = None
name: str | None = None
platform_name: str | None = None
platform_code: str | None = None
status: int | None = None
class Address(_SdsModel):
id: int | None = None
country: str | None = None
consignee: str | None = None
city: str | None = None
detail: str | None = None
province: str | None = None
mobile_phone: str | None = None
postcode: str | None = None
email: str | None = None
class IssuingBayArea(_SdsModel):
name: str | None = None
type: str | None = None
id: int | None = None
class IssuingBayRef(_SdsModel):
id: int | None = None
name: str | None = None
city: str | None = None
province: str | None = None
detail: str | None = None
class ImportProduct(_SdsModel):
id: str | None = None
import_sku: str | None = None
import_product_name: str | None = None
import_color: str | None = None
import_size: str | None = None
import_img: str | None = None
import_style_number: str | None = None
import_sku_id: str | None = None
class OrderItem(_SdsModel):
id: str | None = None
order_id: str | None = None
no: str | None = None
out_order_item_id: str | None = None
import_product: ImportProduct | None = None
product_id: int | None = None
factory_id: int | None = None
factory: Any | None = None
seller_sku: str | None = None
product_name: str | None = None
img_url: str | None = None
product_color_block: Any | None = None
product_size: str | None = None
num: int | None = None
status: int | None = None
origin_price: float | None = None
price: float | None = None
supply_price: float | None = None
amount: float | None = None
profit: float | None = None
refund_status: str | None = None
design_status: str | None = None
class Merchant(_SdsModel):
id: int | None = None
name: str | None = None
class StatusHistory(_SdsModel):
status: int | None = None
gmt_update_time: str | None = None
class Compensation(_SdsModel):
status: int | None = None
amount: float | None = None
class Order(_SdsModel):
id: str
store_id: int | None = None
customer_id: int | None = None
no: str | None = None
out_order_no: str | None = None
username: str | None = None
origin: str | None = None
total_amount: float | None = None
product_amount: float | None = None
cost_amount: float | None = None
status: int | None = None
online_order_status_name: str | None = None
online_order_status_code: str | None = None
gmt_created: str | None = None
gmt_pay: str | None = None
pay_status: int | None = None
latest_ship_time: Any | None = None
refund_status: int | None = None
be_after_service_order: bool | None = None
fba_type: str | None = None
remark: str | None = None
carriage_name: str | None = None
parcel_list: list[Any] = []
merchant_store: MerchantStore | None = None
address: Address | None = None
issuing_bay_area: IssuingBayArea | None = None
issuing_bay: IssuingBayRef | None = None
items: list[OrderItem] = []
logistics_id: int | None = None
delivery_type: int | None = None
merchant: Merchant | None = None
finish_time: Any | None = None
status_history: list[StatusHistory] = []
compensation: Compensation | None = None
compensation_status: int | None = None
express_type: int | None = None
class OrderPageResponse(_SdsModel):
total_count: int = Field(default=0, alias="totalCount")
current_page: int | None = None
page_size: int | None = None
records: list[Order] = Field(default=[], alias="list")
class OrderStatus(_SdsModel):
name: str | None = None
order_status_code: str | None = None
sort: int | None = None
class OrderStatistics(_SdsModel):
name: str | None = None
order_num: int | None = None
order_status_code: str | None = None
sort: int | None = None
# ----- 产品域(camelCase)-----
class ProductCategory(_SdsModel):
id: int | None = None
name: str | None = None
class ProductSupply(_SdsModel):
factory_id: int | None = None
name: str | None = None
supply_price: str | None = None
max_price: float | None = None
min_price: float | None = None
online_supply_num: int | None = None
class Product(_SdsModel):
id: int
img_url: str | None = None
thumb_img_url: str | None = None
category_id: int | None = None
category_list: list[ProductCategory] = []
on_sale_status: int | None = None
public_status: int | None = None
platform_price: str | None = None
on_sale_price: str | None = None
status: int | None = None
sku: str | None = None
name: str | None = None
english_name: str | None = None
product_supply: list[ProductSupply] = []
class ProductListResponse(_SdsModel):
total_count: int = Field(default=0, alias="totalCount")
records: list[Product] = Field(default=[], alias="list")
# ----- 物流域(混用 snake_case,显式 alias)-----
class IssuingBay(_SdsModel):
id: int | None = None
name: str | None = None
type: str | None = None
country: str | None = None
country_name: str | None = Field(default=None, alias="country_name")
province: str | None = None
province_code: str | None = Field(default=None, alias="province_code")
city: str | None = None
city_code: str | None = Field(default=None, alias="city_code")
detail: str | None = None
priority: int | None = None
status: int | None = None
issuing_bay_area_id: int | None = Field(default=None, alias="issuing_bay_area_id")
created_time: int | None = Field(default=None, alias="created_time")
updated_time: int | None = Field(default=None, alias="updated_time")
class IssuingBayAreaDetail(_SdsModel):
id: int | None = None
name: str | None = None
type: str | None = None
country_code: str | None = None
address: str | None = None
icon_url: str | None = None
status: int | None = None
priority: int | None = None
created_time: int | None = Field(default=None, alias="created_time")
updated_time: int | None = Field(default=None, alias="updated_time")
issuing_bays: list[IssuingBay] = []
class IssuingBayAreaResponse(_SdsModel):
items: list[IssuingBayAreaDetail] = []
class LogisticsItem(IssuingBay):
factory_num: int | None = None
account_num: int | None = None
class LogisticsPageResponse(_SdsModel):
items: list[LogisticsItem] = []
page: int | None = None
size: int | None = None
total_count: int = Field(default=0, alias="totalCount")
total_count_alt: int | None = Field(default=None, alias="total_count")
# ----- 筛选面板(camelCase)-----
class FilterItem(_SdsModel):
name: str | None = None
id: Any | None = None
fp: Any | None = None
class FiltratePanelResponse(_SdsModel):
factory_list: list[FilterItem] = []
merchant_list: list[FilterItem] = []
logistics_list: list[FilterItem] = []
early_warning_list: list[FilterItem] = []
issuing_bay_area_list: list[FilterItem] = []
extend_status_list: list[FilterItem] = []
# ----- 查询参数(请求 schema)-----
class OrderPageRequest(BaseModel):
"""SDS 订单分页查询参数。keywords(换行分隔订单号)/时间范围/状态等可选。"""
model_config = ConfigDict(extra="allow")
size: int | None = None
page: int | None = None
keywords: str | None = None
created_date_begin: str | None = None
created_date_end: str | None = None
pay_date_begin: str | None = None
pay_date_end: str | None = None
confirm_date_begin: str | None = None
confirm_date_end: str | None = None
finish_date_begin: str | None = None
finish_date_end: str | None = None
order_status_code: str | None = None
product_keywords: str | None = None
order_time_range_type: str | None = None
factory_ids: list[int] | None = None
merchant_ids: list[int] | None = None
logistics_ids: list[int] | None = None
issuing_bay_area_id: int | None = None
extend_status: str | None = None
early_warning_type: str | None = None
item_num_type: str | None = None
limit: int | None = None
all: bool | None = None
class OrderQueryParams(BaseModel):
"""SDS 订单筛选/状态/统计查询参数(不含分页)。"""
model_config = ConfigDict(extra="allow")
created_date_begin: str | None = None
created_date_end: str | None = None
pay_date_begin: str | None = None
pay_date_end: str | None = None
confirm_date_begin: str | None = None
confirm_date_end: str | None = None
finish_date_begin: str | None = None
finish_date_end: str | None = None
order_time_range_type: str | None = None
product_keywords: str | None = None
factory_ids: list[int] | None = None
merchant_ids: list[int] | None = None
logistics_ids: list[int] | None = None
issuing_bay_area_id: int | None = None
extend_status: str | None = None
early_warning_type: str | None = None
item_num_type: str | None = None
class ProductListRequest(BaseModel):
model_config = ConfigDict(extra="allow")
keyword: str | None = None
page: int | None = None
size: int | None = None
public_status: int | None = None
limit: int | None = None
all: bool | None = None
class LogisticsPageRequest(BaseModel):
model_config = ConfigDict(extra="allow")
keyword: str | None = None
delivery_method: str | None = None
issuing_bay_area_id: int
page: int | None = None
size: int | None = None
limit: int | None = None
all: bool | None = None
+15
View File
@@ -0,0 +1,15 @@
services:
prefect-server:
image: prefecthq/prefect:3-python3.12
container_name: prefect-server
command: prefect server start --host 0.0.0.0
environment:
PREFECT_UI_API_URL: http://192.168.124.92:4200/api
ports:
- "4200:4200"
volumes:
- prefect-data:/root/.prefect
restart: unless-stopped
volumes:
prefect-data:
+134
View File
@@ -0,0 +1,134 @@
# 系统设计架构
本项目把两个外部业务系统(RIIN 分销、SDS POD)的 REST API 封装为可复用的 Python 客户端,
再用 Prefect 编排成 ETL 工作流(REST API → PostgreSQL)。
## 设计理念:高内聚、低耦合、关注点分离
核心原则贯穿整个项目:
1. **高内聚** —— 每个模块只承担一个职责。`api/riin/` 只管 RIIN,`api/sds/` 只管 SDS,
`flows/` 只管编排。一个模块内部的代码都围绕同一件事。
2. **低耦合** —— 客户端层(`api/`)零 Prefect 依赖,可独立使用、独立测试;
编排层(`flows/`)只通过客户端的公开方法调用它。两层之间是单向依赖。
3. **不过度抽象** —— 仅在行为稳定且重复时才抽取共享代码。当前 2 个系统,用组合式 helper
而非强制基类;等出现 3+ 个行为高度一致的源,再考虑抽象 Protocol 接口(YAGNI)。
## 分层结构
```
inkreach-prefect/
├── api/ # 客户端层:纯 httpx + pydantic,不 import prefect
│ ├── _common/ # 共享 helper(组合,非强制基类)
│ │ ├── http.py # TokenHttpClient:token 注入 + 401 自动重登
│ │ ├── cache.py # TTLCache:主数据就近缓存(随 client 生命周期)
│ │ ├── concurrency.py # run_concurrent:线程池 + 并发上限
│ │ └── batch.py # parse_id_text / split_lines / chunk
│ ├── riin/ # RIIN 分销平台
│ │ ├── client.py # RiinDisClient:认证/订单/店铺/辅助主数据
│ │ └── models.py # pydantic schema + 枚举
│ └── sds/ # SDS POD 平台
│ ├── client.py # SdsMngClient:认证/订单/产品/物流
│ └── models.py
├── flows/ # 编排层:依赖 api/,使用 prefect
│ ├── riin_orders_etl.py # @task 封装 → @flow: 抽取→转换→落 CSV(后接 PG)
│ ├── sds_orders_etl.py
│ └── shared/load.py # 共享 task:凭据加载 / CSV / PG 写入
└── pyproject.toml # build-system=hatchling,packages=["api"]
```
### 层次边界(关键)
| 层 | 依赖 | 职责 |
|----|------|------|
| `api/` 客户端层 | httpx、pydantic(**禁止** import prefect) | REST 封装、认证、分页、schema |
| `flows/` 编排层 | prefect、`api/`、pandas | 任务编排、重试、缓存、并发、调度、密钥 |
> 客户端层绝不反向依赖 Prefect。这样客户端能在 ETL 之外被复用(脚本、其他服务、单测)。
> 缓存/重试/并发这些**编排能力**留在 flows 层,客户端只保留自身需要的就近缓存(TTLCache)
> 和内部并发(run_concurrent,用于同一次调用内的多页并发)。
## 客户端层设计要点
### TokenHttpClient 基类(`_common/http.py`)
两个系统共享"token 注入 + 401 自动重登"这一稳定行为,故提取为基类(继承)。
差异通过构造参数由子类配置:
| 系统 | base_url | token 头 | 固定头 | 登录 |
|------|----------|---------|--------|------|
| RIIN | tshirt.riin.com | `Authorization` | X-Time-Zone / X-Requested-With | username + MD5(password) |
| SDS | pod-api.sdspod.com | `access-token` | 无 | username + password + 公司名 |
子类只需实现 `_login()`,其余 token 生命周期(缓存/失效/重试)由基类统一处理。
### 模式还原(忠实移植自 TS SDK)
- **分页三态**:单页 / `limit`(取首页 N 条)/ `all`(全量并发分页)
- **批量切分**:RIIN 订单 OID/快递单号 >500 自动分批;SDS 订单 keywords >200 自动分批
- **就近缓存**:主数据(店铺/工厂/客户/统计)用 TTLCache 缓存 ~5 分钟
- **并发**:多页/多批用 `run_concurrent`(线程池,默认 10)
### Schema
每个系统用 pydantic v2 建模参数与返回(`extra='allow'` 不丢字段)。
- RIIN 全 camelCase → `alias_generator=to_camel`。
- SDS 订单/产品 camelCase;物流域混用 snake_case,对 snake_case 字段显式 `Field(alias=...)`。
注意:响应字段名避免用 Python 内置名(如 `list` 会与 `list[Order]` 注解冲突,改用 `records` + `alias="list"`)。
## 编排层设计要点(对照 Prefect 文档)
| TS SDK 概念 | Prefect 对应 | 位置 |
|------------|-------------|------|
| 401 自动重试 | `@task(retries=3, retry_delay_seconds=[2,5,15])` | flows/*.py 的 extract task |
| 主数据缓存 | task 级 `cache_key_fn`(按需启用) | flows |
| 并发限制器 | Prefect 并发限制 / `.submit()`+`.map()` | flows |
| 凭据 | Prefect Secret Block(加密) | flows 内部读取 |
任务粒度遵循 Prefect"小而可观测"原则:extract / transform / load 各为独立 task。
凭据在 extract task **内部**读取 Secret,不作为参数传递,避免泄露到日志/UI。
## 密钥管理
- **PostgreSQL**:`SqlAlchemyConnector` Block,名 `pg-inkreach-dmp`。
- **RIIN / SDS**:JSON 存入 `Secret` Block,名 `riin-credentials` / `sds-credentials`,
值为 `{"username":"...","password":"..."}`。
- 本地开发备用:`.env`(已 gitignore)。
## 扩展性:如何接入第三个系统
加法式,不碰现有代码(开闭原则):
1. `api/<新系统>/` 新增 `client.py`(继承 TokenHttpClient)+ `models.py`。
2. `flows/<新系统>_orders_etl.py` 新增一个 flow,复用 `flows/shared/load.py`。
3. 注册该系统的凭据 Secret Block。
若新系统行为与现有高度一致,可白嫖 `api/_common/` 的 helper;差异大就自包含实现。
暂不强制统一 `Connector` 接口 —— 系统异质(REST 不同认证 vs DB vs 文件),强行抽象会变成漏抽象。
等行为趋同后再抽 Protocol。
## 同步任务层(flows/riin、flows/sds)
对齐 `inkreach-bi-auth-mng` 的 sync tasks,每系统 6 个,效果一致:
| 任务 | RIIN | SDS | 说明 |
|------|------|-----|------|
| 时间窗口同步 | `order_sync` | `order_sync` | 按时间范围(+筛选)拉取 |
| 增量同步 | `incremental_sync` | `incremental_sync` | 从 DB `MAX(orderedAt)` 增量,无参 |
| 近期同步 | `recent_sync` | `recent_sync` | 近 N 天 |
| 刷新未完结 | `refresh_pending` | `refresh_pending` | DB 未完结订单号批量刷新 |
| 指定单号/100前缀 | `by_orders` | `refresh_100_prefix` | RIIN 按单号;SDS 替换 100 前缀暂存 |
| 数据核验 | `verify` | `verify` | 逐天比对远程/本地计数并修复差异 |
### 共享编排层(flows/shared)
- `mappers.py`:RIIN/SDS 原始订单 → 统一 `OrderUpsertInput` / `GoodsUpsertInput`,含状态/平台/快递来源/国家归一映射。
- `db.py`:`SyncDB.sync_orders_and_goods()` —— Orders/Goods 批量 `ON CONFLICT` upsert,**写入前用库中已有 `createdAt` 覆盖**(保证幂等),Goods 写前删除该订单不再出现的行;另有 `get_latest_ordered_at` / `get_pending_orders` / `get_order_count_by_status` / `get_local_order_nos` / `get_local_goods_count` / `get_orders_with_100_prefix` 等查询。
- `clients.py`:客户端工厂(读 Secret Block)、`fetch_riin/sds_all_pages`(顺序分页,页间 500ms,仅对 5xx/429/超时重试)、RIIN 商品并发拉取(并发 20)、SDS 物流渠道字典(24h 缓存)。
### 关键差异点
- RIIN 订单与商品分离(需 `getGoods` 逐单拉取,并发 20);SDS 订单内联 `items`(直接映射,无需额外请求)。
- SDS 同步默认**跳过待付款**(refresh_pending 除外);RIIN 无此过滤。
- 两个系统时间维度不同:RIIN 用 `timeType=3`(商家推送)+ `startTime/endTime`;SDS 用 `payDate` 或 `createdDate`。
- `verify` 用各自的统计接口(RIIN `getCount`、SDS `getStatus`)按状态比对,排除 SDS 的"售后"。
+1
View File
@@ -0,0 +1 @@
"""RIIN 同步 flows:order_sync / incremental_sync / recent_sync / refresh_pending / by_orders / verify。"""
+45
View File
@@ -0,0 +1,45 @@
"""RIIN 同步公共管道:映射订单 → 并发拉商品 → upsert 到 PG。"""
from __future__ import annotations
from typing import Any
from flows.shared import fetch_goods_concurrent
from flows.shared.db import SyncDB
from flows.shared.mappers import map_riin_goods, map_riin_order
from api import RiinDisClient
def sync_riin_records(raw_orders: list[Any], client: RiinDisClient, db: SyncDB) -> dict[str, int]:
order_inputs: list[Any] = []
order_meta: dict[str, tuple[str, Any]] = {} # raw.id -> (orderNo, createdAt)
goods_failed = 0
for raw in raw_orders:
try:
o = map_riin_order(raw)
except Exception as err: # noqa: BLE001
print(f"[RIIN] 映射订单失败 {getattr(raw, 'platform_oid', '?')}: {err}")
continue
order_inputs.append(o)
if raw.id:
order_meta[raw.id] = (o.orderNo, o.createdAt)
goods_inputs: list[Any] = []
if order_meta:
goods_map = fetch_goods_concurrent(client, list(order_meta.keys()))
for oid, (order_no, created_at) in order_meta.items():
items = goods_map.get(oid, [])
if not items:
goods_failed += 1
for g in items:
try:
goods_inputs.append(map_riin_goods(g, order_no, created_at))
except Exception as err: # noqa: BLE001
print(f"[RIIN] 映射商品失败 order={order_no}: {err}")
stats = db.sync_orders_and_goods(order_inputs, goods_inputs)
stats["orderCount"] = len(order_inputs)
stats["goodsCount"] = len(goods_inputs)
stats["goodsFetchFailed"] = goods_failed
return stats
+59
View File
@@ -0,0 +1,59 @@
"""RIIN 指定单号同步:通过订单号列表(逗号/换行分隔)拉取指定订单。
对应 riin-by-orders。
"""
from __future__ import annotations
import re
from prefect import flow, task
from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records
from flows.shared import SyncDB, create_riin_client
def _split_order_nos(text: str) -> list[str]:
seen: dict[str, None] = {}
for part in re.split(r"[\n\r,,\s]+", text or ""):
part = part.strip()
if part and part not in seen:
seen[part] = None
return list(seen.keys())
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def fetch_by_orders(order_nos: list[str]) -> list:
client = create_riin_client()
try:
resp = client.get_orders(OrderPageRequest(platform_oid_list=",".join(order_nos), all=True))
return list(resp.data.records)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_riin_client()
try:
return sync_riin_records(raw_orders, client, SyncDB())
finally:
client.close()
@flow(name="riin-by-orders-sync", log_prints=True)
def riin_by_orders_sync(order_nos: str) -> dict:
nos = _split_order_nos(order_nos)
print(f"RIIN 指定单号同步 输入 {len(nos)} 个")
if not nos:
return {"inputCount": 0, "matchedCount": 0}
raw = fetch_by_orders(nos)
stats = upsert_records(raw)
print(f"完成: 匹配 {len(raw)} | {stats}")
return {"inputCount": len(nos), "matchedCount": len(raw), **stats}
if __name__ == "__main__":
riin_by_orders_sync.serve(name="riin-by-orders-sync")
+60
View File
@@ -0,0 +1,60 @@
"""RIIN 增量同步:从数据库最近下单时间增量拉取(无参,全自动)。
对应 riin-incremental-sync。from = floorToHour(MAX(orderedAt)),to = now。
"""
from __future__ import annotations
from datetime import datetime
from prefect import flow, task
from flows.riin._pipeline import sync_riin_records
from flows.shared import (
SyncDB, create_riin_client, fetch_riin_all_pages, floor_to_hour, format_date,
)
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict) -> list:
client = create_riin_client()
try:
return fetch_riin_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_riin_client()
try:
return sync_riin_records(raw_orders, client, SyncDB())
finally:
client.close()
@flow(name="riin-incremental-sync", log_prints=True)
def riin_incremental_sync() -> dict:
db = SyncDB()
latest = db.get_latest_ordered_at("RIIN")
now = datetime.now()
from_dt = floor_to_hour(latest) if latest else floor_to_hour(now)
to_dt = now
if from_dt >= to_dt:
from_dt = floor_to_hour(now)
params = {
"time_type": 3,
"start_time": format_date(from_dt),
"end_time": format_date(to_dt),
"page_size": 200,
}
print(f"RIIN 增量同步 {params['start_time']} ~ {params['end_time']}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, **stats}
if __name__ == "__main__":
riin_incremental_sync.serve(name="riin-incremental-sync", cron="0 * * * *")
+77
View File
@@ -0,0 +1,77 @@
"""RIIN 订单同步:按时间窗口(+可选筛选)拉取订单与商品。
对应 riin-order-sync。timeType 固定 3(商家推送时间)。
"""
from __future__ import annotations
from typing import Any
from prefect import flow, task
from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records
from flows.shared import SyncDB, create_riin_client, fetch_riin_all_pages, format_date
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict[str, Any]) -> list[Any]:
client = create_riin_client()
try:
return fetch_riin_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list[Any]) -> dict[str, int]:
client = create_riin_client()
try:
return sync_riin_records(raw_orders, client, SyncDB())
finally:
client.close()
@flow(name="riin-order-sync", log_prints=True)
def riin_order_sync(
order_date_begin: str,
order_date_end: str,
platform_type: int | None = None,
order_status: int | None = None,
factory_ids: list[str] | None = None,
express_source: int | None = None,
urgent_order: int | None = None,
has_refund: bool | None = None,
page_size: int = 200,
) -> dict[str, Any]:
if not order_date_begin or not order_date_end:
raise ValueError("order_date_begin / order_date_end 为必填")
params: dict[str, Any] = {
"time_type": 3,
"start_time": order_date_begin,
"end_time": order_date_end,
"page_size": page_size,
}
if platform_type is not None:
params["platform_type"] = platform_type
if order_status is not None:
params["order_status"] = order_status
if factory_ids:
params["factory_ids"] = factory_ids
if express_source is not None:
params["express_source"] = express_source
if urgent_order is not None:
params["urgent_order"] = urgent_order
if has_refund is not None:
params["has_refund"] = has_refund
print(f"RIIN 订单同步 {order_date_begin} ~ {order_date_end}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} | {stats}")
return {"syncRange": {"from": order_date_begin, "to": order_date_end}, **stats}
if __name__ == "__main__":
riin_order_sync.serve(name="riin-order-sync")
+56
View File
@@ -0,0 +1,56 @@
"""RIIN 近期同步:拉取最近 N 天订单(默认 7 天,1~90)。
对应 riin-recent-sync。from = startOfDay(now - N days),to = now。
"""
from __future__ import annotations
from datetime import datetime, timedelta
from prefect import flow, task
from flows.riin._pipeline import sync_riin_records
from flows.shared import (
SyncDB, create_riin_client, fetch_riin_all_pages, format_date, start_of_day,
)
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict) -> list:
client = create_riin_client()
try:
return fetch_riin_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_riin_client()
try:
return sync_riin_records(raw_orders, client, SyncDB())
finally:
client.close()
@flow(name="riin-recent-sync", log_prints=True)
def riin_recent_sync(recent_days: int = 7) -> dict:
days = min(max(recent_days, 1), 90)
now = datetime.now()
from_dt = start_of_day(now - timedelta(days=days))
params = {
"time_type": 3,
"start_time": format_date(from_dt),
"end_time": format_date(now),
"page_size": 200,
}
print(f"RIIN 近期同步 近 {days} 天 {params['start_time']} ~ {params['end_time']}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
return {"syncRange": {"from": params["start_time"], "to": params["end_time"]}, "recentDays": days, **stats}
if __name__ == "__main__":
riin_recent_sync.serve(name="riin-recent-sync", cron="30 * * * *")
+58
View File
@@ -0,0 +1,58 @@
"""RIIN 刷新未完结:查 DB 中未完结订单号,按 platformOidList 批量(500)刷新。
对应 riin-refresh-pending。终态:已退款 / 已关闭 / 已发货。
"""
from __future__ import annotations
from prefect import flow, task
from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records
from flows.shared import SyncDB, create_riin_client
RIIN_FINISHED_STATUSES = ["已退款", "已关闭", "已发货"]
ORDER_BATCH = 500
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def fetch_pending(order_nos: list[str]) -> list:
client = create_riin_client()
try:
all_raw: list = []
for i in range(0, len(order_nos), ORDER_BATCH):
batch = order_nos[i : i + ORDER_BATCH]
resp = client.get_orders(
OrderPageRequest(platform_oid_list=",".join(batch), all=True)
)
all_raw.extend(resp.data.records)
return all_raw
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_riin_client()
try:
return sync_riin_records(raw_orders, client, SyncDB())
finally:
client.close()
@flow(name="riin-refresh-pending", log_prints=True)
def riin_refresh_pending() -> dict:
db = SyncDB()
order_nos = db.get_pending_orders("RIIN", RIIN_FINISHED_STATUSES)
print(f"RIIN 未完结订单 {len(order_nos)} 个")
if not order_nos:
return {"pendingCount": 0, "matchedCount": 0}
raw = fetch_pending(order_nos)
stats = upsert_records(raw)
print(f"完成: 匹配 {len(raw)} | {stats}")
return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats}
if __name__ == "__main__":
riin_refresh_pending.serve(name="riin-refresh-pending", cron="0 */3 * * *")
+126
View File
@@ -0,0 +1,126 @@
"""RIIN 数据核验:逐天比对远程/本地按状态计数,有差异则拉取修复。
对应 riin-verify。mode: recent(近 N 天,不含今天)/ range(指定区间)。
"""
from __future__ import annotations
from datetime import datetime, timedelta
from prefect import flow, task
from api.riin.models import OrderPageRequest
from flows.riin._pipeline import sync_riin_records
from flows.shared import (
SyncDB, create_riin_client, fetch_riin_all_pages, format_date, start_of_day,
)
RIIN_COUNT_STATUS_MAP = {
"assignedSuceessed": "已分配",
"factoryAuditing": "工厂审核中",
"inProducing": "生产中",
"shipped": "已发货",
"closed": "已关闭",
"refunding": "退款中",
"refunded": "已退款",
"rejecting": "拒审中",
"rejectedToShop": "已退回店铺",
}
def _parse_days(mode: str, recent_days: int, date_range: str | None) -> list[datetime]:
today = start_of_day(datetime.now())
if mode == "range" and date_range:
parts = [p.strip() for p in date_range.split("~")]
from_dt = datetime.fromisoformat(parts[0])
to_dt = datetime.fromisoformat(parts[1]) if len(parts) > 1 and parts[1] else from_dt
days = []
cur = from_dt
while cur <= to_dt:
days.append(cur)
cur += timedelta(days=1)
return days
days_n = min(max(recent_days, 1), 90)
return [today - timedelta(days=i) for i in range(days_n, 0, -1)]
def _count_to_status_map(data: dict) -> dict[str, int]:
out: dict[str, int] = {}
for key, count in data.items():
if key.startswith("dayRange"):
continue
label = RIIN_COUNT_STATUS_MAP.get(key)
if label and count and count > 0:
out[label] = count
return out
def _diff(remote: dict[str, int], local: dict[str, int]) -> tuple[dict[str, int], bool]:
diff: dict[str, int] = {}
has_diff = False
for status in set(remote) | set(local):
r = remote.get(status, 0)
l = local.get(status, 0)
if r != l:
diff[status] = l - r
has_diff = True
return diff, has_diff
@flow(name="riin-verify", log_prints=True)
def riin_verify(mode: str = "recent", recent_days: int = 1, date_range: str | None = None) -> dict:
days = _parse_days(mode, recent_days, date_range)
db = SyncDB()
client = create_riin_client()
totals = {k: 0 for k in ("ordersAdded", "ordersUpdated", "ordersRemoved",
"goodsAdded", "goodsUpdated", "goodsRemoved")}
days_with_diff = 0
day_reports = []
try:
for day in days:
day_str = format_date(day).split(" ")[0]
day_start = start_of_day(day)
day_end = day_start + timedelta(days=1)
count_resp = client.get_order_count(
OrderPageRequest(time_type=3, start_time=format_date(day_start), end_time=format_date(day_end))
)
remote_map = _count_to_status_map(count_resp.data.model_dump(by_alias=False))
remote_total = sum(remote_map.values())
local_counts = db.get_order_count_by_status("RIIN", day_start, day_end)
local_goods = db.get_local_goods_count("RIIN", day_start, day_end)
diff, has_diff = _diff(remote_map, local_counts)
print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}"
+ (f" 差异 {diff}" if has_diff else " 一致,跳过"))
if not has_diff:
day_reports.append({"date": day_str, "diff": diff, "ok": True})
continue
days_with_diff += 1
raw_orders = fetch_riin_all_pages(
client,
{"time_type": 3, "start_time": format_date(day_start), "end_time": format_date(day_end), "page_size": 200},
)
stats = sync_riin_records(raw_orders, client, db)
for k in ("ordersCreated", "ordersUpdated", "goodsCreated", "goodsUpdated"):
pass
totals["ordersAdded"] += stats["ordersCreated"]
totals["ordersUpdated"] += stats["ordersUpdated"]
totals["goodsAdded"] += stats["goodsCreated"]
totals["goodsUpdated"] += stats["goodsUpdated"]
day_reports.append({"date": day_str, "diff": diff, "ok": False,
"orders": stats["orderCount"], "goods": stats["goodsCount"]})
finally:
client.close()
return {
"verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}},
"syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""},
}
if __name__ == "__main__":
riin_verify.serve(name="riin-verify", cron="0 6 * * *")
+1
View File
@@ -0,0 +1 @@
"""SDS 同步 flows:order_sync / incremental_sync / recent_sync / refresh_pending / refresh_100_prefix / verify。"""
+58
View File
@@ -0,0 +1,58 @@
"""SDS 同步公共管道:映射订单(含内联 items)→ upsert 到 PG。"""
from __future__ import annotations
from typing import Any, Callable
from flows.shared import resolve_courier_source
from flows.shared.db import SyncDB
from flows.shared.mappers import map_sds_goods, map_sds_order
def map_sds_records(
raw_orders: list[Any],
channel_map: dict[int, str | None],
channel_refresh: Callable[[], dict[int, str | None]] | None = None,
skip_unpaid: bool = True,
) -> tuple[list[Any], list[Any], int]:
"""映射 SDS 订单+商品。返回 (order_inputs, goods_inputs, skipped_unpaid)。"""
order_inputs: list[Any] = []
goods_inputs: list[Any] = []
refreshed: set[int] = set()
skipped_unpaid = 0
for raw in raw_orders:
courier = None
if channel_refresh is not None:
courier = resolve_courier_source(raw.logistics_id, channel_map, channel_refresh, refreshed)
try:
o = map_sds_order(raw, channel_map, courier)
except Exception as err: # noqa: BLE001
print(f"[SDS] 映射订单失败 {getattr(raw, 'no', '?')}: {err}")
continue
if skip_unpaid and o.systemStatus == "待付款":
skipped_unpaid += 1
continue
order_inputs.append(o)
for item in raw.items or []:
try:
goods_inputs.append(map_sds_goods(item, o.orderNo, o.createdAt))
except Exception as err: # noqa: BLE001
print(f"[SDS] 映射商品失败 order={o.orderNo}: {err}")
return order_inputs, goods_inputs, skipped_unpaid
def sync_sds_records(
raw_orders: list[Any], channel_map: dict[int, str | None], db: SyncDB,
channel_refresh: Callable[[], dict[int, str | None]] | None = None,
skip_unpaid: bool = True,
) -> dict[str, int]:
order_inputs, goods_inputs, skipped = map_sds_records(
raw_orders, channel_map, channel_refresh, skip_unpaid
)
stats = db.sync_orders_and_goods(order_inputs, goods_inputs)
stats["orderCount"] = len(order_inputs)
stats["goodsCount"] = len(goods_inputs)
stats["skippedUnpaid"] = skipped
return stats
+57
View File
@@ -0,0 +1,57 @@
"""SDS 增量同步:从数据库最近下单时间增量拉取(无参,全自动)。
对应 sds-incremental-sync。from = floorToHour(MAX(orderedAt)),to = now,按 payDate。
"""
from __future__ import annotations
from datetime import datetime
from prefect import flow, task
from flows.sds._pipeline import sync_sds_records
from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, floor_to_hour, format_date,
get_channel_type_map, refresh_channel_type_map,
)
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict) -> list:
client = create_sds_client()
try:
return fetch_sds_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_sds_client()
try:
channel_map = get_channel_type_map(client)
return sync_sds_records(raw_orders, channel_map, SyncDB(),
channel_refresh=lambda: refresh_channel_type_map(client))
finally:
client.close()
@flow(name="sds-incremental-sync", log_prints=True)
def sds_incremental_sync() -> dict:
db = SyncDB()
latest = db.get_latest_ordered_at("SDS")
now = datetime.now()
from_dt = floor_to_hour(latest) if latest else floor_to_hour(now)
if from_dt >= now:
from_dt = floor_to_hour(now)
params = {"pay_date_begin": format_date(from_dt), "pay_date_end": format_date(now), "size": 500}
print(f"SDS 增量同步 {params['pay_date_begin']} ~ {params['pay_date_end']}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
return {"syncRange": {"from": params["pay_date_begin"], "to": params["pay_date_end"]}, **stats}
if __name__ == "__main__":
sds_incremental_sync.serve(name="sds-incremental-sync", cron="15 * * * *")
+76
View File
@@ -0,0 +1,76 @@
"""SDS 订单同步:按下单时间窗口(+筛选)拉取订单与商品。
对应 sds-order-sync。主时间维度 payDate(下单时间)。
"""
from __future__ import annotations
from typing import Any
from prefect import flow, task
from flows.sds._pipeline import sync_sds_records
from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, get_channel_type_map, refresh_channel_type_map,
)
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict[str, Any]) -> list:
client = create_sds_client()
try:
return fetch_sds_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_sds_client()
try:
channel_map = get_channel_type_map(client)
return sync_sds_records(raw_orders, channel_map, SyncDB(),
channel_refresh=lambda: refresh_channel_type_map(client))
finally:
client.close()
@flow(name="sds-order-sync", log_prints=True)
def sds_order_sync(
pay_date_begin: str,
pay_date_end: str,
order_status_code: str | None = None,
factory_ids: list[int] | None = None,
merchant_ids: list[int] | None = None,
logistics_ids: list[int] | None = None,
issuing_bay_area_id: int | None = None,
keywords: str | None = None,
page_size: int = 500,
) -> dict[str, Any]:
params: dict[str, Any] = {
"pay_date_begin": pay_date_begin,
"pay_date_end": pay_date_end,
"size": page_size,
}
if order_status_code is not None:
params["order_status_code"] = order_status_code
if factory_ids:
params["factory_ids"] = factory_ids
if merchant_ids:
params["merchant_ids"] = merchant_ids
if logistics_ids:
params["logistics_ids"] = logistics_ids
if issuing_bay_area_id is not None:
params["issuing_bay_area_id"] = issuing_bay_area_id
if keywords:
params["keywords"] = keywords
print(f"SDS 订单同步 {pay_date_begin} ~ {pay_date_end}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']} / 跳过待付款 {stats.get('skippedUnpaid',0)}")
return {"syncRange": {"from": pay_date_begin, "to": pay_date_end}, **stats}
if __name__ == "__main__":
sds_order_sync.serve(name="sds-order-sync")
+56
View File
@@ -0,0 +1,56 @@
"""SDS 近期同步:拉取最近 N 天订单(默认 7,1~90),按 createdDate。
对应 sds-recent-sync。from = startOfDay(now - N days),to = startOfDay(today)。
"""
from __future__ import annotations
from datetime import datetime, timedelta
from prefect import flow, task
from flows.sds._pipeline import sync_sds_records
from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, format_date, get_channel_type_map,
refresh_channel_type_map, start_of_day,
)
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def extract_orders(params: dict) -> list:
client = create_sds_client()
try:
return fetch_sds_all_pages(client, params)
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_sds_client()
try:
channel_map = get_channel_type_map(client)
return sync_sds_records(raw_orders, channel_map, SyncDB(),
channel_refresh=lambda: refresh_channel_type_map(client))
finally:
client.close()
@flow(name="sds-recent-sync", log_prints=True)
def sds_recent_sync(recent_days: int = 7) -> dict:
days = min(max(recent_days, 1), 90)
now = datetime.now()
from_dt = start_of_day(now - timedelta(days=days))
to_dt = start_of_day(now)
params = {"created_date_begin": format_date(from_dt), "created_date_end": format_date(to_dt), "size": 500}
print(f"SDS 近期同步 近 {days} 天 {params['created_date_begin']} ~ {params['created_date_end']}")
raw = extract_orders(params)
stats = upsert_records(raw)
print(f"完成: 订单 {stats['orderCount']} / 商品 {stats['goodsCount']}")
return {"syncRange": {"from": params["created_date_begin"], "to": format_date(now)},
"recentDays": days, **stats}
if __name__ == "__main__":
sds_recent_sync.serve(name="sds-recent-sync", cron="45 * * * *")
+104
View File
@@ -0,0 +1,104 @@
"""SDS 100前缀刷新:备货中/待确认且 productionNo 以 100 开头的订单,工厂分配后用新数据替换。
对应 sds-refresh-100-prefix。
"""
from __future__ import annotations
from prefect import flow, task
from api.sds.models import OrderPageRequest
from flows.shared import (
SyncDB, create_sds_client, get_channel_type_map,
refresh_channel_type_map, resolve_courier_source,
)
from flows.shared.mappers import map_sds_goods, map_sds_order
TARGET_STATUSES = ["备货中", "待确认"]
KEYWORDS_BATCH = 200
def _has_100_prefix_item(raw) -> bool:
for item in raw.items or []:
prod_no = (item.no or item.id) if hasattr(item, "no") else None
if prod_no and str(prod_no).startswith("100"):
return True
return False
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def fetch_remote(order_nos: list[str]) -> dict:
client = create_sds_client()
try:
remote: dict[str, object] = {}
for i in range(0, len(order_nos), KEYWORDS_BATCH):
batch = order_nos[i : i + KEYWORDS_BATCH]
resp = client.get_orders(OrderPageRequest(keywords="\n".join(batch), all=True))
for o in resp.records:
if o.no:
remote[o.no] = o
return remote
finally:
client.close()
@task
def upsert_records(remote_map: dict, target_order_nos: list[str]) -> dict:
client = create_sds_client()
try:
channel_map = get_channel_type_map(client)
refreshed: set[int] = set()
order_inputs: list = []
goods_inputs: list = []
skipped_pending = skipped_unpaid = 0
for no in target_order_nos:
raw = remote_map.get(no)
if not raw:
continue
if _has_100_prefix_item(raw):
skipped_pending += 1
continue
courier = resolve_courier_source(raw.logistics_id, channel_map,
lambda: refresh_channel_type_map(client), refreshed)
try:
o = map_sds_order(raw, channel_map, courier)
except Exception as err: # noqa: BLE001
print(f"[SDS 100] 映射失败 {no}: {err}")
continue
if o.systemStatus == "待付款":
skipped_unpaid += 1
continue
order_inputs.append(o)
for item in raw.items or []:
try:
goods_inputs.append(map_sds_goods(item, o.orderNo, o.createdAt))
except Exception: # noqa: BLE001
pass
stats = SyncDB().sync_orders_and_goods(order_inputs, goods_inputs)
stats["skippedStillPending"] = skipped_pending
stats["skippedUnpaid"] = skipped_unpaid
return stats
finally:
client.close()
@flow(name="sds-refresh-100-prefix", log_prints=True)
def sds_refresh_100_prefix() -> dict:
db = SyncDB()
targets = db.get_orders_with_100_prefix("SDS", TARGET_STATUSES)
order_nos = list({o for o, _ in targets})
print(f"SDS 100前缀目标订单 {len(order_nos)} 个")
if not order_nos:
return {"targetCount": 0, "refreshed": 0}
remote = fetch_remote(order_nos)
stats = upsert_records(remote, order_nos)
print(f"完成: 刷新 {stats['orderCount']} / 仍待分配 {stats.get('skippedStillPending',0)} / 待付款 {stats.get('skippedUnpaid',0)}")
return {"targetCount": len(order_nos), **stats}
if __name__ == "__main__":
sds_refresh_100_prefix.serve(name="sds-refresh-100-prefix", cron="30 */2 * * *")
+61
View File
@@ -0,0 +1,61 @@
"""SDS 刷新未完结:查 DB 未完结订单号,按 keywords 批量刷新。
对应 sds-refresh-pending。终态:已完成 / 已取消。注意不跳过待付款。
"""
from __future__ import annotations
from prefect import flow, task
from api.sds.models import OrderPageRequest
from flows.sds._pipeline import sync_sds_records
from flows.shared import (
SyncDB, create_sds_client, get_channel_type_map, refresh_channel_type_map,
)
SDS_FINISHED_STATUSES = ["已完成", "已取消"]
KEYWORDS_BATCH = 500
@task(retries=3, retry_delay_seconds=[3, 6, 9])
def fetch_pending(order_nos: list[str]) -> list:
client = create_sds_client()
try:
all_raw: list = []
for i in range(0, len(order_nos), KEYWORDS_BATCH):
batch = order_nos[i : i + KEYWORDS_BATCH]
resp = client.get_orders(OrderPageRequest(keywords="\n".join(batch), all=True))
all_raw.extend(resp.records)
return all_raw
finally:
client.close()
@task
def upsert_records(raw_orders: list) -> dict:
client = create_sds_client()
try:
channel_map = get_channel_type_map(client)
return sync_sds_records(raw_orders, channel_map, SyncDB(),
channel_refresh=lambda: refresh_channel_type_map(client),
skip_unpaid=False)
finally:
client.close()
@flow(name="sds-refresh-pending", log_prints=True)
def sds_refresh_pending() -> dict:
db = SyncDB()
order_nos = db.get_pending_orders("SDS", SDS_FINISHED_STATUSES)
print(f"SDS 未完结订单 {len(order_nos)} 个")
if not order_nos:
return {"pendingCount": 0, "matchedCount": 0}
raw = fetch_pending(order_nos)
stats = upsert_records(raw)
print(f"完成: 匹配 {len(raw)} | {stats}")
return {"pendingCount": len(order_nos), "matchedCount": len(raw), **stats}
if __name__ == "__main__":
sds_refresh_pending.serve(name="sds-refresh-pending", cron="0 */2 * * *")
+104
View File
@@ -0,0 +1,104 @@
"""SDS 数据核验:逐天比对远程/本地按状态计数(排除"售后"),有差异则拉取修复。
对应 sds-verify。mode: recent(近 N 天,不含今天)/ range(指定区间)。
"""
from __future__ import annotations
from datetime import datetime, timedelta
from prefect import flow, task
from api.sds.models import OrderQueryParams
from flows.sds._pipeline import sync_sds_records
from flows.shared import (
SyncDB, create_sds_client, fetch_sds_all_pages, format_date, get_channel_type_map,
refresh_channel_type_map, start_of_day,
)
def _parse_days(mode: str, recent_days: int, date_range: str | None) -> list[datetime]:
today = start_of_day(datetime.now())
if mode == "range" and date_range:
parts = [p.strip() for p in date_range.split("~")]
from_dt = datetime.fromisoformat(parts[0])
to_dt = datetime.fromisoformat(parts[1]) if len(parts) > 1 and parts[1] else from_dt
days = []
cur = from_dt
while cur <= to_dt:
days.append(cur)
cur += timedelta(days=1)
return days
n = min(max(recent_days, 1), 90)
return [today - timedelta(days=i) for i in range(n, 0, -1)]
def _diff(remote_stats: list, local: dict[str, int]) -> tuple[dict[str, int], bool]:
diff: dict[str, int] = {}
has_diff = False
statuses = {getattr(s, "name", None) for s in remote_stats} | set(local)
statuses.discard("售后")
statuses.discard(None)
for status in statuses:
remote = next((getattr(s, "orderNum", 0) for s in remote_stats if getattr(s, "name", None) == status), 0)
local_n = local.get(status, 0)
if remote != local_n:
diff[status] = local_n - remote
has_diff = True
return diff, has_diff
@flow(name="sds-verify", log_prints=True)
def sds_verify(mode: str = "recent", recent_days: int = 1, date_range: str | None = None) -> dict:
days = _parse_days(mode, recent_days, date_range)
db = SyncDB()
client = create_sds_client()
channel_map = get_channel_type_map(client)
totals = {k: 0 for k in ("ordersAdded", "ordersUpdated", "goodsAdded", "goodsUpdated")}
days_with_diff = 0
day_reports = []
try:
for day in days:
day_str = format_date(day).split(" ")[0]
day_start = start_of_day(day)
day_end = day_start + timedelta(days=1)
remote_stats = client.get_status(
OrderQueryParams(pay_date_begin=format_date(day_start), pay_date_end=format_date(day_end))
)
remote_total = sum(getattr(s, "orderNum", 0) or 0 for s in remote_stats)
local_counts = db.get_order_count_by_status("SDS", day_start, day_end)
diff, has_diff = _diff(remote_stats, local_counts)
print(f"[核验 {day_str}] 远程 {remote_total} 本地 {sum(local_counts.values())}"
+ (f" 差异 {diff}" if has_diff else " 一致,跳过"))
if not has_diff:
day_reports.append({"date": day_str, "diff": diff, "ok": True})
continue
days_with_diff += 1
raw = fetch_sds_all_pages(
client,
{"pay_date_begin": format_date(day_start), "pay_date_end": format_date(day_end), "size": 500},
)
stats = sync_sds_records(raw, channel_map, db,
channel_refresh=lambda: refresh_channel_type_map(client))
totals["ordersAdded"] += stats["ordersCreated"]
totals["ordersUpdated"] += stats["ordersUpdated"]
totals["goodsAdded"] += stats["goodsCreated"]
totals["goodsUpdated"] += stats["goodsUpdated"]
day_reports.append({"date": day_str, "diff": diff, "ok": False,
"orders": stats["orderCount"], "goods": stats["goodsCount"]})
finally:
client.close()
return {
"verifyReport": {"days": day_reports, "summary": {"totalDays": len(days), "daysWithDiff": days_with_diff, **totals}},
"syncRange": {"from": format_date(start_of_day(days[0])) if days else "", "to": format_date(start_of_day(days[-1])) if days else ""},
}
if __name__ == "__main__":
sds_verify.serve(name="sds-verify", cron="0 7 * * *")
+27
View File
@@ -0,0 +1,27 @@
"""同步编排共享层:客户端/分页、映射、PG 写入。"""
from flows.shared import clients, db, mappers
from flows.shared.clients import (
create_riin_client,
create_sds_client,
fetch_goods_concurrent,
fetch_riin_all_pages,
fetch_sds_all_pages,
floor_to_hour,
format_date,
get_channel_type_map,
refresh_channel_type_map,
resolve_courier_source,
start_of_day,
)
from flows.shared.db import SyncDB
from flows.shared.mappers import GoodsUpsertInput, OrderUpsertInput
__all__ = [
"clients", "db", "mappers", "SyncDB",
"OrderUpsertInput", "GoodsUpsertInput",
"create_riin_client", "create_sds_client",
"fetch_riin_all_pages", "fetch_sds_all_pages", "fetch_goods_concurrent",
"get_channel_type_map", "refresh_channel_type_map", "resolve_courier_source",
"format_date", "start_of_day", "floor_to_hour",
]
+229
View File
@@ -0,0 +1,229 @@
"""同步共享:客户端工厂、顺序分页(含重试/延迟)、RIIN 商品拉取、SDS 物流渠道字典、日期辅助。
行为对齐 inkreach-bi-auth-mng 的 sync/tasks/clients.ts:
- 顺序分页(页间 500ms),非并发,对外部 API 友好。
- fetch_with_retry:仅对 5xx / 429 / 超时重试,最多 3 次,退避 3s*次数。
"""
from __future__ import annotations
import json
import time
from datetime import datetime, timedelta, timezone
from typing import Any, Callable, TypeVar
from prefect.blocks.system import Secret
from api import RiinDisClient, SdsMngClient
from api.riin.models import OrderPageRequest as RiinReq
from api.sds.models import LogisticsPageRequest, OrderPageRequest as SdsReq
from flows.shared.mappers import map_sds_channel_type
T = TypeVar("T")
GOODS_CONCURRENCY = 20
PAGE_DELAY_MS = 500
MAX_RETRIES = 3
RETRY_DELAY_MS = 3000
TZ_CN = timezone(timedelta(hours=8))
_RIIN_FINISHED = ["已退款", "已关闭", "已发货"]
_SDS_FINISHED = ["已完成", "已取消"]
# ---------- 客户端工厂 ----------
def create_riin_client() -> RiinDisClient:
cred = _load_secret("riin-credentials")
return RiinDisClient(cred["username"], cred["password"])
def create_sds_client() -> SdsMngClient:
cred = _load_secret("sds-credentials")
return SdsMngClient(cred["username"], cred["password"])
def _load_secret(name: str) -> dict[str, str]:
value = Secret.load(name).get()
if isinstance(value, str):
value = json.loads(value)
return value
# ---------- 日期辅助 ----------
def format_date(d: datetime) -> str:
return d.strftime("%Y-%m-%d %H:%M:%S")
def start_of_day(d: datetime) -> datetime:
return d.replace(hour=0, minute=0, second=0, microsecond=0)
def floor_to_hour(d: datetime) -> datetime:
return d.replace(minute=0, second=0, microsecond=0)
# ---------- 重试 ----------
def fetch_with_retry(fn: Callable[[], T], label: str) -> T:
last_err: Exception | None = None
for attempt in range(1, MAX_RETRIES + 1):
try:
return fn()
except Exception as err: # noqa: BLE001
last_err = err
status = _status_of(err)
msg = str(err)
is_timeout = "timeout" in msg.lower() or "timed out" in msg.lower()
retryable = status == 429 or (status is not None and status >= 500) or is_timeout
if attempt < MAX_RETRIES and retryable:
print(f"[sync] {label} 第 {attempt}/{MAX_RETRIES} 次重试 ({status}: {msg[:120]})")
time.sleep(RETRY_DELAY_MS * attempt / 1000)
continue
raise
assert last_err is not None
raise last_err
def _status_of(err: Exception) -> int | None:
for attr in ("status_code", "statusCode", "status"):
v = getattr(err, attr, None)
if isinstance(v, int):
return v
resp = getattr(err, "response", None)
return getattr(resp, "status_code", None) if resp is not None else None
# ---------- 分页 ----------
def fetch_riin_all_pages(client: RiinDisClient, params: dict[str, Any], max_pages: int = 1000) -> list[Any]:
page_size = params.get("page_size", 200)
base = {**params, "page_size": page_size}
def fetch(idx: int):
return fetch_with_retry(lambda: client.get_orders(RiinReq(**{**base, "page_index": idx})), f"RIIN page {idx}")
first = fetch(1)
total = first.data.total
records = list(first.data.records)
total_pages = min(-(-total // page_size), max_pages)
for page in range(2, total_pages + 1):
resp = fetch(page)
records.extend(resp.data.records)
time.sleep(PAGE_DELAY_MS / 1000)
if total_pages == max_pages:
print(f"[RIIN sync] 达到最大页数限制 {max_pages},截断到 {len(records)} 条")
return records
def fetch_sds_all_pages(client: SdsMngClient, params: dict[str, Any]) -> list[Any]:
page_size = params.get("size", 500)
base = {**params, "size": page_size}
def fetch(idx: int):
return fetch_with_retry(lambda: client.get_orders(SdsReq(**{**base, "page": idx})), f"SDS page {idx}")
first = fetch(1)
records = list(first.records)
total_pages = -(-first.total_count // page_size)
for page in range(2, total_pages + 1):
resp = fetch(page)
records.extend(resp.records)
time.sleep(PAGE_DELAY_MS / 1000)
return records
# ---------- RIIN 商品 ----------
def fetch_goods_for_order(client: RiinDisClient, order_id: str) -> list[Any]:
try:
resp = client.get_order_goods(order_id)
return list(resp.data)
except Exception as err: # noqa: BLE001
print(f"[RIIN sync] 订单 {order_id} 商品拉取失败: {err}")
return []
def fetch_goods_concurrent(client: RiinDisClient, order_ids: list[str]) -> dict[str, list[Any]]:
"""以 GOODS_CONCURRENCY 并发拉取多订单商品,返回 {order_id: [goods...]}。"""
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
results: dict[str, list[Any]] = {}
if not order_ids:
return results
def make(oid: str) -> Callable[[], None]:
def run() -> None:
results[oid] = fetch_goods_for_order(client, oid)
return run
with ThreadPoolExecutor(max_workers=min(GOODS_CONCURRENCY, len(order_ids))) as pool:
list(pool.map(lambda fn: fn(), [make(oid) for oid in order_ids]))
return results
# ---------- SDS 物流渠道字典 ----------
_channel_cache: dict[str, Any] = {}
_CHANNEL_TTL = 24 * 3600
_CHANNEL_KEY = "all"
def get_channel_type_map(client: SdsMngClient) -> dict[int, str | None]:
cached = _channel_cache.get(_CHANNEL_KEY)
if cached and cached["expire_at"] > time.time():
return cached["data"]
mapping: dict[int, str | None] = {}
try:
areas = client.get_logistics_areas()
area_ids = [a.id for a in areas.items if a.id is not None]
except Exception as err: # noqa: BLE001
print(f"[SDS sync] 物流区域拉取失败: {err}")
area_ids = []
for area_id in area_ids:
try:
resp = client.get_logistics_page(LogisticsPageRequest(issuing_bay_area_id=area_id, all=True, size=200))
for item in resp.items:
if item.id is None:
continue
raw = getattr(item, "channelType", None) or getattr(item, "channel_type", None)
label = map_sds_channel_type(raw)
if label is None:
continue
mapping[int(item.id)] = label
except Exception as err: # noqa: BLE001
print(f"[SDS sync] 区域 {area_id} 物流字典拉取失败: {err}")
_channel_cache[_CHANNEL_KEY] = {"data": mapping, "expire_at": time.time() + _CHANNEL_TTL}
return mapping
def refresh_channel_type_map(client: SdsMngClient) -> dict[int, str | None]:
_channel_cache.pop(_CHANNEL_KEY, None)
return get_channel_type_map(client)
def resolve_courier_source(
raw_logistics_id: Any,
mapping: dict[int, str | None],
refresh: Callable[[], dict[int, str | None]],
refreshed_ids: set[int],
) -> str | None:
if raw_logistics_id is None:
return None
try:
lid = int(raw_logistics_id)
except (TypeError, ValueError):
return None
if lid in mapping:
return mapping[lid]
if lid in refreshed_ids:
return None
refreshed_ids.add(lid)
try:
new_map = refresh()
mapping.update(new_map)
except Exception: # noqa: BLE001
pass
return mapping.get(lid)
+365
View File
@@ -0,0 +1,365 @@
"""PG 同步写入层:Orders / Goods / Merchant 的 upsert + 同步所需的查询。
忠实移植自 inkreach-bi-auth-mng 的 sync/lib/db.service.ts。
- Orders 主键:("orderNo", "createdAt")
- Goods 主键:("orderNo", "productionNo")
关键:写入前用库中已有 createdAt 覆盖,避免同一订单因 createdAt 不同产生重复主键行。
"""
from __future__ import annotations
from datetime import datetime
from typing import Any
from psycopg2.extras import execute_values
from flows.shared.mappers import GoodsUpsertInput, OrderUpsertInput
ORDER_CHUNK = 1000
GOODS_CHUNK = 2000
LOOKUP_BATCH = 10000
class SyncDB:
def __init__(self, block_name: str = "pg-inkreach-dmp") -> None:
from prefect_sqlalchemy import SqlAlchemyConnector
self._engine = SqlAlchemyConnector.load(block_name).get_engine()
def _conn(self):
return self._engine.raw_connection()
# ---------- 查询辅助 ----------
def get_latest_ordered_at(self, system_name: str) -> datetime | None:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT MAX(COALESCE("orderedAt","createdAt")) FROM "Orders" WHERE "systemName"=%s',
(system_name,),
)
row = cur.fetchone()
return row[0] if row and row[0] else None
def get_pending_orders(self, system_name: str, exclude_statuses: list[str]) -> list[str]:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s '
'AND "systemStatus" IS DISTINCT FROM ALL %s',
(system_name, exclude_statuses),
)
rows = cur.fetchall()
return [r[0] for r in rows]
def get_order_count_by_status(self, system_name: str, start: datetime, end: datetime) -> dict[str, int]:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT "systemStatus", COUNT(*) FROM "Orders" WHERE "systemName"=%s '
'AND "orderedAt">=%s AND "orderedAt"<%s GROUP BY "systemStatus"',
(system_name, start, end),
)
rows = cur.fetchall()
return {r[0]: int(r[1]) for r in rows}
def get_local_order_nos(self, system_name: str, start: datetime, end: datetime) -> list[str]:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s '
'AND "orderedAt">=%s AND "orderedAt"<%s',
(system_name, start, end),
)
rows = cur.fetchall()
return [r[0] for r in rows]
def get_local_goods_count(self, system_name: str, start: datetime, end: datetime) -> int:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT COUNT(*) FROM "Goods" g JOIN "Orders" o '
'ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" '
'WHERE o."systemName"=%s AND o."orderedAt">=%s AND o."orderedAt"<%s',
(system_name, start, end),
)
row = cur.fetchone()
return int(row[0]) if row else 0
def get_orders_with_100_prefix(self, system_name: str, statuses: list[str]) -> list[tuple[str, datetime]]:
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT DISTINCT o."orderNo", o."createdAt" FROM "Orders" o '
'JOIN "Goods" g ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" '
'WHERE o."systemName"=%s AND o."systemStatus"=ANY(%s) AND g."productionNo" LIKE \'100%\'',
(system_name, statuses),
)
rows = cur.fetchall()
return [(r[0], r[1]) for r in rows]
# ---------- 主 upsert ----------
def sync_orders_and_goods(
self,
orders: list[OrderUpsertInput],
goods: list[GoodsUpsertInput],
delete_order_nos: list[str] | None = None,
) -> dict[str, int]:
result = {
"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0,
"goodsUpdated": 0, "goodsRemoved": 0, "ordersDeleted": 0,
}
if not orders and not goods and not delete_order_nos:
return result
conn = self._conn()
try:
system_name = orders[0].systemName if orders else None
all_order_nos = list({o.orderNo for o in orders} | {g.orderNo for g in goods})
# 1. 解析已有 createdAt,覆盖到 orders(保证幂等 upsert)
existing_created: dict[str, datetime] = {}
for i in range(0, len(all_order_nos), LOOKUP_BATCH):
chunk = all_order_nos[i : i + LOOKUP_BATCH]
with conn.cursor() as cur:
cur.execute(
'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)'
+ (f' AND "systemName"=%s' if system_name else ""),
(chunk, system_name) if system_name else (chunk,),
)
for r in cur.fetchall():
existing_created[r[0]] = r[1]
for o in orders:
if o.orderNo in existing_created:
o.createdAt = existing_created[o.orderNo]
# 2. 去重 orders / goods
seen_o: set[str] = set()
orders_dedup: list[OrderUpsertInput] = []
for o in orders:
key = f'{o.orderNo}|{o.createdAt.isoformat()}'
if key in seen_o:
continue
seen_o.add(key)
orders_dedup.append(o)
resolved_created: dict[str, datetime] = {}
for o in orders_dedup:
resolved_created.setdefault(o.orderNo, o.createdAt)
seen_g: set[str] = set()
goods_dedup: list[GoodsUpsertInput] = []
for g in goods:
if g.orderNo in resolved_created:
g.createdAt = resolved_created[g.orderNo]
key = f'{g.orderNo}|{g.productionNo}'
if key in seen_g:
continue
seen_g.add(key)
goods_dedup.append(g)
goods_by_order: dict[str, list[GoodsUpsertInput]] = {}
for g in goods_dedup:
goods_by_order.setdefault(g.orderNo, []).append(g)
# 3. 分批 upsert orders + 关联 goods
for i in range(0, len(orders_dedup), ORDER_CHUNK):
chunk = orders_dedup[i : i + ORDER_CHUNK]
related: list[GoodsUpsertInput] = []
for o in chunk:
related.extend(goods_by_order.get(o.orderNo, []))
stats = self._upsert_order_batch(conn, chunk, related, system_name)
for k, v in stats.items():
result[k] += v
# 4. 剩余 goods(其 order 不在本次 orders 列表)
orders_set = {o.orderNo for o in orders_dedup}
remaining = [g for g in goods_dedup if g.orderNo not in orders_set]
if remaining:
c, u = self._upsert_goods_only(conn, remaining, system_name)
result["goodsCreated"] += c
result["goodsUpdated"] += u
# 5. 删除订单
if delete_order_nos:
with conn.cursor() as cur:
cur.execute(
'DELETE FROM "Orders" WHERE "orderNo"=ANY(%s)'
+ (f' AND "systemName"=%s' if system_name else ""),
(delete_order_nos, system_name) if system_name else (delete_order_nos,),
)
result["ordersDeleted"] = cur.rowcount
# 6. 商家占位
self._ensure_merchants(conn, orders_dedup)
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
return result
def _upsert_order_batch(
self, conn, orders: list[OrderUpsertInput], goods: list[GoodsUpsertInput], system_name: str | None
) -> dict[str, int]:
stats = {"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0, "goodsUpdated": 0, "goodsRemoved": 0}
with conn.cursor() as cur:
# 统计已存在 → created/updated
pairs = [(o.orderNo, o.createdAt) for o in orders]
cur.execute(
"SELECT t.order_no, t.created_at FROM unnest(%s::text[], %s::timestamp[]) AS t(order_no,created_at) "
"JOIN \"Orders\" o ON o.\"orderNo\"=t.order_no AND o.\"createdAt\"=t.created_at",
_unnest_pairs(pairs),
)
existing_set = {(r[0], r[1]) for r in cur.fetchall()}
updated = sum(1 for o in orders if (o.orderNo, o.createdAt) in existing_set)
stats["ordersUpdated"] = updated
stats["ordersCreated"] = len(orders) - updated
execute_values(
cur,
'''INSERT INTO "Orders" (
"orderNo","outOrderNo","courierSource","courierName","courierNo","merchantId",
"countryRegion","factoryName","systemName","systemStatus","platformName","platformStatus",
"createdAt","orderedAt","confirmedAt","completedAt","shipDeadline"
) VALUES %s
ON CONFLICT ("orderNo","createdAt") DO UPDATE SET
"outOrderNo"=EXCLUDED."outOrderNo","courierSource"=EXCLUDED."courierSource",
"courierName"=EXCLUDED."courierName","courierNo"=EXCLUDED."courierNo",
"merchantId"=EXCLUDED."merchantId","countryRegion"=EXCLUDED."countryRegion",
"factoryName"=EXCLUDED."factoryName","systemName"=EXCLUDED."systemName",
"systemStatus"=EXCLUDED."systemStatus","platformName"=EXCLUDED."platformName",
"platformStatus"=EXCLUDED."platformStatus","orderedAt"=EXCLUDED."orderedAt",
"confirmedAt"=EXCLUDED."confirmedAt","completedAt"=EXCLUDED."completedAt",
"shipDeadline"=EXCLUDED."shipDeadline"''',
[(
o.orderNo, o.outOrderNo, o.courierSource, o.courierName, o.courierNo, o.merchantId,
o.countryRegion, o.factoryName, o.systemName, o.systemStatus, o.platformName, o.platformStatus,
o.createdAt, o.orderedAt, o.confirmedAt, o.completedAt, o.shipDeadline,
) for o in orders],
page_size=ORDER_CHUNK,
)
# 关联 goods:先删本次不再出现的,再 upsert
if goods:
order_nos = list({g.orderNo for g in goods})
prod_nos = [g.productionNo for g in goods]
cur.execute(
'''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s)
AND ("orderNo","createdAt") IN (
SELECT "orderNo","createdAt" FROM "Orders"
WHERE "orderNo"=ANY(%s)''' + (f' AND "systemName"=%s' if system_name else "") + ')',
(order_nos, prod_nos, order_nos, system_name) if system_name else (order_nos, prod_nos, order_nos),
)
stats["goodsRemoved"] += cur.rowcount
g_pairs = [(g.orderNo, g.productionNo) for g in goods]
cur.execute(
"SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) "
"JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no",
_unnest_pairs(g_pairs),
)
g_existing = {(r[0], r[1]) for r in cur.fetchall()}
g_updated = sum(1 for g in goods if (g.orderNo, g.productionNo) in g_existing)
stats["goodsUpdated"] = g_updated
stats["goodsCreated"] = len(goods) - g_updated
execute_values(
cur,
'''INSERT INTO "Goods" (
"orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor",
"itemSize","itemStyleNo","quantity","status"
) VALUES %s
ON CONFLICT ("orderNo","productionNo") DO UPDATE SET
"createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo",
"itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor",
"itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo",
"quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''',
[(
g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor,
g.itemSize, g.itemStyleNo, g.quantity, g.status,
) for g in goods],
page_size=GOODS_CHUNK,
)
return stats
def _upsert_goods_only(
self, conn, goods: list[GoodsUpsertInput], system_name: str | None
) -> tuple[int, int]:
# 用库中实际 createdAt 覆盖
order_nos = list({g.orderNo for g in goods})
actual: dict[str, datetime] = {}
with conn.cursor() as cur:
cur.execute(
'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)'
+ (f' AND "systemName"=%s' if system_name else ""),
(order_nos, system_name) if system_name else (order_nos,),
)
for r in cur.fetchall():
actual[r[0]] = r[1]
for g in goods:
if g.orderNo in actual:
g.createdAt = actual[g.orderNo]
created = updated = 0
for i in range(0, len(goods), GOODS_CHUNK):
chunk = goods[i : i + GOODS_CHUNK]
with conn.cursor() as cur:
c_order = list({g.orderNo for g in chunk})
c_prod = [g.productionNo for g in chunk]
cur.execute(
'''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s)'''
+ (f''' AND ("orderNo","createdAt") IN (SELECT "orderNo","createdAt" FROM "Orders"
WHERE "orderNo"=ANY(%s) AND "systemName"=%s)''' if system_name else ""),
(c_order, c_prod, c_order, system_name) if system_name else (c_order, c_prod),
)
g_pairs = [(g.orderNo, g.productionNo) for g in chunk]
cur.execute(
"SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) "
"JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no",
_unnest_pairs(g_pairs),
)
g_existing = {(r[0], r[1]) for r in cur.fetchall()}
u = sum(1 for g in chunk if (g.orderNo, g.productionNo) in g_existing)
updated += u
created += len(chunk) - u
execute_values(
cur,
'''INSERT INTO "Goods" (
"orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor",
"itemSize","itemStyleNo","quantity","status"
) VALUES %s
ON CONFLICT ("orderNo","productionNo") DO UPDATE SET
"createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo",
"itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor",
"itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo",
"quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''',
[(
g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor,
g.itemSize, g.itemStyleNo, g.quantity, g.status,
) for g in chunk],
page_size=GOODS_CHUNK,
)
return created, updated
def _ensure_merchants(self, conn, orders: list[OrderUpsertInput]) -> None:
ids = list({o.merchantId for o in orders if o.merchantId and o.merchantId.strip()})
if not ids:
return
with conn.cursor() as cur:
execute_values(
cur,
'''INSERT INTO "Merchant" ("merchantId","merchantGroup","responsibleName","responsibleId","tags")
VALUES %s ON CONFLICT ("merchantId") DO NOTHING''',
[(mid, "", None, None, "") for mid in ids],
)
def _unnest_pairs(pairs: list[tuple]) -> tuple[list, list]:
"""构造 psycopg2 unnest 双数组参数:返回 (col_a_list, col_b_list)。
配合 SQL:unnest(%s::text[], %s::timestamp[]) 或 unnest(%s::text[], %s::text[])。
"""
return [p[0] for p in pairs], [p[1] for p in pairs]
+317
View File
@@ -0,0 +1,317 @@
"""数据映射:RIIN / SDS 原始订单 → 统一 Order/Goods 写入格式。
忠实移植自 inkreach-bi-auth-mng 的 sync/mappers/{riin,sds,normalize-country}.ts。
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any
TZ_CN = timezone(timedelta(hours=8))
# ---------- RIIN 状态 / 平台 / 快递来源映射 ----------
RIIN_SYSTEM_STATUS_MAP: dict[int, str] = {
-4: "待推送", -3: "工厂反审", -5: "待审核", -6: "分配失败",
1: "待推送", 3: "已反审", 4: "生产中", 5: "生产中",
12: "已发货", 13: "已关闭", 14: "已取消", 15: "已关闭",
}
RIIN_PLATFORM_TYPE_MAP: dict[int, str] = {
2: "淘宝", 3: "拼多多", 6: "Shopee", 7: "LAZADA", 8: "1688",
11: "TIKTOK", 13: "TEMU", 15: "自建站(海外)", 16: "SHOPIFY",
17: "SHEIN", 18: "自建站(国内)", 19: "WALMART", 21: "AMAZON",
26: "OZON", 27: "Wildberries", 30: "ETSY", 31: "MERCADO",
}
RIIN_EXPRESS_SOURCE_MAP: dict[int, str] = {
1: "工厂发货", 2: "平台面单", 3: "自有面单", 4: "分销面单", 5: "电商面单",
}
RIIN_GOODS_STATUS_MAP: dict[str, str] = {
"NOT_SHIPPED": "待发货", "SHIPPED": "已发货", "CANCEL": "已取消",
"CANCELLED": "已取消", "REFUNDING": "退款中",
}
RIIN_PLATFORM_STATUS_MAP: dict[str, str] = {
"SHIPPED": "已发货", "NOT_SHIPPED": "待发货", "CANCEL": "已取消",
"PAUSE": "暂停", "COMPLETE": "已完成",
}
# ---------- SDS 状态 / 物流渠道映射 ----------
SDS_SYSTEM_STATUS_MAP: dict[int, str] = {
1: "待付款", 2: "待确认", 3: "备货中", 4: "已完成", 5: "搁置中",
7: "待设计", 9: "部分发货", 98: "已取消",
}
SDS_GOODS_STATUS_MAP: dict[int, str] = {**SDS_SYSTEM_STATUS_MAP, 99: "已删除"}
SDS_CHANNEL_TYPE_LABEL_MAP: dict[str, str] = {
"online": "线上物流", "offline": "普通物流", "manual": "手动物流", "self_pickup": "自提",
}
# ---------- 日期解析 ----------
def safe_date(value: Any, context: str = "") -> datetime:
"""解析日期字符串(无时区视为 +08:00),返回 tz-aware datetime。"""
if not value or not isinstance(value, str):
raise ValueError(f"Invalid date for {context}: {value!r}")
s = value.strip()
if "+" not in s and "T" not in s and not s.endswith("Z"):
s = f"{s}T00:00:00+08:00" if len(s) == 10 else f"{s}+08:00"
try:
dt = datetime.fromisoformat(s)
except ValueError:
raise ValueError(f"Invalid date for {context}: {value!r}")
return dt if dt.tzinfo else dt.replace(tzinfo=TZ_CN)
def _from_epoch_ms(value: Any) -> datetime | None:
try:
n = int(value)
except (TypeError, ValueError):
return None
return datetime.fromtimestamp(n / 1000, tz=TZ_CN) if n > 0 else None
# ---------- 国家/地区归一 ----------
_US_ALIASES = {"US", "UNITED STATES", "USA", "ESTADOS UNIDOS", "美国加州发货仓"}
_MX_ALIASES = {"MX", "MEXICO", "MÉXICO", "MEX", "ME"}
_CA_ALIASES = {"加拿大工厂", "CA"}
_PL_ALIASES = {"欧州波兰发货仓", "欧洲波兰法兰克福发货仓"}
_WAREHOUSE_MAP = {
"印美达澳洲发货仓": "澳洲", "欧洲德国发货仓": "德国", "欧洲匈牙利发货仓": "匈牙利",
"欧洲西班牙发货仓": "西班牙", "欧洲意大利发货仓": "意大利",
}
_DIRECT_MAP = {"Saudi Arabia": "沙特", "Japan": "日本"}
_EXCLUDED_COUNTRIES = {
"Croatia", "Czech Republic", "丹麦", "保加利亚", "克罗地亚", "卢森堡", "奥地利", "希腊",
"拉脱维亚", "捷克", "斯洛伐克", "斯洛文尼亚", "比利时", "法国", "爱尔兰", "爱沙尼亚",
"瑞典", "瑞士", "立陶宛", "罗马尼亚", "芬兰", "荷兰", "Denmark", "Austria", "Greece",
"Latvia", "Slovakia", "Slovenia", "Belgium", "France", "Ireland", "Estonia", "Sweden",
"Switzerland", "Lithuania", "Romania", "Finland", "Netherlands", "Luxembourg", "Bulgaria",
}
def normalize_country_region(raw: str | None, factory_name: str | None = None) -> str | None:
if factory_name == "广州印美达":
return "中国"
if not raw:
return None
trimmed = raw.strip()
if trimmed in _EXCLUDED_COUNTRIES:
return None
upper = trimmed.upper()
if upper in _US_ALIASES:
return "美国"
if upper in _MX_ALIASES:
return "墨西哥"
if upper in _CA_ALIASES:
return "加拿大"
if trimmed in _PL_ALIASES:
return "波兰"
if trimmed in _WAREHOUSE_MAP:
return _WAREHOUSE_MAP[trimmed]
if raw in _DIRECT_MAP:
return _DIRECT_MAP[raw]
return raw
# ---------- 统一写入模型 ----------
class OrderUpsertInput:
__slots__ = (
"orderNo", "outOrderNo", "courierSource", "courierName", "courierNo",
"merchantId", "countryRegion", "factoryName", "systemName", "systemStatus",
"platformName", "platformStatus", "createdAt", "orderedAt", "confirmedAt",
"completedAt", "shipDeadline",
)
def __init__(self, **kwargs: Any) -> None:
for k in self.__slots__:
setattr(self, k, kwargs.get(k))
class GoodsUpsertInput:
__slots__ = (
"orderNo", "productionNo", "createdAt", "outOrderNo", "itemName",
"itemColor", "itemSize", "itemStyleNo", "quantity", "status",
)
def __init__(self, **kwargs: Any) -> None:
for k in self.__slots__:
setattr(self, k, kwargs.get(k))
# ---------- RIIN 映射 ----------
def map_riin_system_status(order_status: int) -> str | None:
return RIIN_SYSTEM_STATUS_MAP.get(order_status)
def map_riin_order(raw: Any) -> OrderUpsertInput:
platform_oid = raw.platform_oid
ordered_at = _parse(raw.push_time, f"RIIN order {platform_oid} pushTime")
confirmed_at = _parse(raw.distributor_push_time, f"RIIN order {platform_oid} distributorPushTime")
system_status = map_riin_system_status(raw.order_status) if raw.order_status is not None else None
is_terminal = system_status in ("已发货", "已关闭", "已取消")
completed_at: datetime | None = None
if is_terminal:
completed_at = _parse(raw.shipping_time, "") or _parse(raw.finish_time, "")
ship_deadline = _parse(raw.except_last_shipping_time, "")
courier_source = RIIN_EXPRESS_SOURCE_MAP.get(raw.express_source) if raw.express_source is not None else None
factory_name = raw.distributor_push_target_name
return OrderUpsertInput(
orderNo=platform_oid,
outOrderNo=raw.source_platform_oid,
courierSource=courier_source,
courierName=raw.delivery_courier,
courierNo=raw.courier_number,
merchantId=raw.sub_name,
countryRegion=normalize_country_region(raw.receiver_country, factory_name),
factoryName=factory_name,
systemName="RIIN",
systemStatus=system_status,
platformName=RIIN_PLATFORM_TYPE_MAP.get(raw.platform_type, str(raw.platform_type)) if raw.platform_type is not None else None,
platformStatus=RIIN_PLATFORM_STATUS_MAP.get(raw.platform_order_status, raw.platform_order_status) if raw.platform_order_status else None,
createdAt=safe_date(raw.order_create_time, f"RIIN order {platform_oid} orderCreateTime"),
orderedAt=ordered_at,
confirmedAt=confirmed_at,
completedAt=completed_at,
shipDeadline=ship_deadline,
)
def map_riin_goods(raw: Any, parent_order_no: str, parent_created_at: datetime) -> GoodsUpsertInput:
product_code = raw.product_code
item_name = product_code.split("-")[0] if product_code else None
return GoodsUpsertInput(
orderNo=parent_order_no,
productionNo=raw.id,
createdAt=parent_created_at,
outOrderNo=raw.platform_oid,
itemName=item_name,
itemColor=raw.color_code,
itemSize=raw.size_code,
itemStyleNo=item_name,
quantity=raw.num if raw.num is not None else 1,
status=RIIN_GOODS_STATUS_MAP.get(raw.goods_status) if raw.goods_status else None,
)
# ---------- SDS 映射 ----------
def map_sds_channel_type(channel_type: Any) -> str | None:
if not isinstance(channel_type, str):
return None
return SDS_CHANNEL_TYPE_LABEL_MAP.get(channel_type)
def map_sds_system_status(status: int) -> str | None:
return SDS_SYSTEM_STATUS_MAP.get(status)
def map_sds_order(raw: Any, channel_type_map: dict[int, str | None], pre_courier: str | None = None) -> OrderUpsertInput:
ordered_at = _parse(raw.gmt_pay, f"SDS order {raw.no} gmtPay")
system_status = map_sds_system_status(raw.status) if raw.status is not None else None
is_terminal = system_status in ("已完成", "已取消")
completed_at: datetime | None = None
if is_terminal and raw.finish_time and raw.finish_time > 0:
completed_at = _from_epoch_ms(raw.finish_time)
ship_deadline = _from_epoch_ms(raw.latest_ship_time) if raw.latest_ship_time else None
courier_source = pre_courier
if courier_source is None and raw.logistics_id is not None:
try:
lid = int(raw.logistics_id)
except (TypeError, ValueError):
lid = None
if lid is not None and lid in channel_type_map:
courier_source = channel_type_map[lid]
courier_no: str | None = None
if raw.parcel_list:
first = raw.parcel_list[0]
try:
courier_no = first["logistics"]["carriage"]["no"]
except (TypeError, KeyError, IndexError):
courier_no = None
factory_name: str | None = None
for item in raw.items or []:
fname = getattr(getattr(item, "factory", None), "name", None)
if not fname and isinstance(item, dict):
fname = (item.get("factory") or {}).get("name")
if fname:
factory_name = fname
break
area_name = raw.issuing_bay_area.name if raw.issuing_bay_area else None
return OrderUpsertInput(
orderNo=raw.no,
outOrderNo=raw.out_order_no,
courierSource=courier_source,
courierName=raw.carriage_name,
courierNo=courier_no,
merchantId=raw.merchant.name if raw.merchant else None,
countryRegion=normalize_country_region(area_name, factory_name),
factoryName=factory_name,
systemName="SDS",
systemStatus=system_status,
platformName=raw.merchant_store.platform_name if raw.merchant_store else None,
platformStatus=raw.online_order_status_name,
createdAt=safe_date(raw.gmt_created, f"SDS order {raw.no}"),
orderedAt=ordered_at,
confirmedAt=None,
completedAt=completed_at,
shipDeadline=ship_deadline,
)
def _parse_sds_style_no(name: str | None) -> str | None:
if not name:
return None
parts = name.split("()")
if len(parts) < 3:
return None
subs = parts[2].split("-")
if len(subs) == 2:
return subs[0] or None
if len(subs) >= 3:
return subs[1] or None
return None
def map_sds_goods(raw: Any, parent_order_no: str, parent_created_at: datetime) -> GoodsUpsertInput:
product_name = raw.product_name
color_name = None
block = raw.product_color_block
if block is not None:
color_name = getattr(block, "color_name", None) or (block.get("colorName") if isinstance(block, dict) else None)
return GoodsUpsertInput(
orderNo=parent_order_no,
productionNo=raw.no or raw.id,
createdAt=parent_created_at,
outOrderNo=raw.out_order_item_id,
itemName=product_name,
itemColor=color_name,
itemSize=raw.product_size,
itemStyleNo=_parse_sds_style_no(product_name),
quantity=raw.num if raw.num is not None else 1,
status=SDS_GOODS_STATUS_MAP.get(raw.status) if raw.status is not None else None,
)
def _parse(value: Any, context: str) -> datetime | None:
if not value:
return None
if isinstance(value, (int, float)):
return _from_epoch_ms(value)
try:
return safe_date(value, context)
except ValueError:
return None
+15
View File
@@ -0,0 +1,15 @@
{
"$schema": "https://opencode.ai/config.json",
"mcp": {
"prefect": {
"type": "local",
"command": [
"/root/.local/bin/uvx",
"--from",
"prefect-mcp",
"prefect-mcp-server"
],
"enabled": true
}
}
}
+20
View File
@@ -0,0 +1,20 @@
[project]
name = "inkreach-prefect"
version = "0.1.0"
description = "ETL workflows orchestrated by Prefect"
requires-python = ">=3.12"
dependencies = [
"prefect>=3.4",
"prefect-sqlalchemy",
"psycopg2-binary",
"httpx",
"pandas",
"pydantic>=2.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["api"]
+39
View File
@@ -0,0 +1,39 @@
"""一键部署全部 12 个同步 flow(单进程注册所有 deployment)。
运行后,所有 flow + deployment 立即出现在 Prefect UI,并按各自 cron 调度执行:
uv run python serve_all.py
不想立即跑调度?部署后在 UI 的 Deployments 页面暂停(pause)对应 deployment 即可,
flow 仍会显示在 Flows 列表中。
"""
from prefect import serve
from flows.riin.by_orders import riin_by_orders_sync
from flows.riin.incremental_sync import riin_incremental_sync
from flows.riin.order_sync import riin_order_sync
from flows.riin.recent_sync import riin_recent_sync
from flows.riin.refresh_pending import riin_refresh_pending
from flows.riin.verify import riin_verify
from flows.sds.incremental_sync import sds_incremental_sync
from flows.sds.order_sync import sds_order_sync
from flows.sds.recent_sync import sds_recent_sync
from flows.sds.refresh_100_prefix import sds_refresh_100_prefix
from flows.sds.refresh_pending import sds_refresh_pending
from flows.sds.verify import sds_verify
if __name__ == "__main__":
serve(
riin_order_sync.to_deployment(name="riin-order-sync"),
riin_incremental_sync.to_deployment(name="riin-incremental-sync", cron="0 * * * *"),
riin_recent_sync.to_deployment(name="riin-recent-sync", cron="30 * * * *"),
riin_refresh_pending.to_deployment(name="riin-refresh-pending", cron="0 */3 * * *"),
riin_by_orders_sync.to_deployment(name="riin-by-orders-sync"),
sds_order_sync.to_deployment(name="sds-order-sync"),
sds_incremental_sync.to_deployment(name="sds-incremental-sync", cron="15 * * * *"),
sds_recent_sync.to_deployment(name="sds-recent-sync", cron="45 * * * *"),
sds_refresh_pending.to_deployment(name="sds-refresh-pending", cron="0 */2 * * *"),
sds_refresh_100_prefix.to_deployment(name="sds-refresh-100-prefix", cron="30 */2 * * *"),
riin_verify.to_deployment(name="riin-verify", cron="0 6 * * *"),
sds_verify.to_deployment(name="sds-verify", cron="0 7 * * *"),
)
Generated
+2279
View File
File diff suppressed because it is too large Load Diff