yeuimu
4a7c44a354
CI / test (pull_request) Successful in 1m11s
- api/riin/client.py: 新增 get_order_goods_batch(distribution/order/goods/list) - flows/shared/clients.py: fetch_goods_concurrent → fetch_goods_for_orders, 批量 200/批、串行、批间 2s,批级重试;移除逐单全局限速(340~500ms)机制 - flows/riin/_pipeline.py: 适配新批量拉取 - flows/shared/db.py: 存在性预查 unnest cast timestamp[] → timestamptz[] (入参为带时区偏移的 tz-aware datetime,timestamptz 才能稳定匹配) 背景:旧逐单接口高频被软限流(成功返回空 data),10-28 起日均缺口 13~33%; 批量接口调用量从日均 ~1 万次降到 ~50 次,已实测回补 7689 单/9717 条商品全量成功。
631 lines
30 KiB
Python
631 lines
30 KiB
Python
"""PG 同步写入层:Orders / Goods / Merchant 的 upsert + 同步所需的查询。
|
|
|
|
忠实移植自 inkreach-bi-auth-mng 的 sync/lib/db.service.ts。
|
|
- Orders 主键:("orderNo", "createdAt")
|
|
- Goods 主键:("orderNo", "productionNo")
|
|
关键:写入前用库中已有 createdAt 覆盖,避免同一订单因 createdAt 不同产生重复主键行。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
from typing import Any, Callable
|
|
|
|
from psycopg2 import errors as pg_errors
|
|
from psycopg2.extras import execute_values
|
|
|
|
from flows.shared.mappers import GoodsUpsertInput, OrderUpsertInput
|
|
|
|
ORDER_CHUNK = 1000
|
|
GOODS_CHUNK = 2000
|
|
LOOKUP_BATCH = 10000
|
|
|
|
# 写 Orders/Goods 的互斥:所有写事务先取同一把 advisory 锁(事务级,commit/rollback 即释放)。
|
|
# 单锁无环,写事务之间从结构上不可能死锁。1768851058 = "inkr" 的 ASCII 整数表示。
|
|
DB_WRITE_LOCK_KEY = 1768851058
|
|
LOCK_TIMEOUT = "15s" # 等锁超时 → LockNotAvailable → 交给重试兜底
|
|
WRITE_RETRIES = 5 # 死锁/等锁超时的整事务重试次数(外部写者如 17track
|
|
# 物流回写不走 advisory 锁,冲突窗口可达分钟级,重试需够深)
|
|
WRITE_RETRY_BACKOFF = [1, 3, 7, 15, 30] # 重试退避(秒;末位预留,最后一次失败直接抛出)
|
|
WRITE_RETRYABLE_ERRORS = (pg_errors.DeadlockDetected, pg_errors.LockNotAvailable)
|
|
|
|
|
|
class SyncDB:
|
|
def __init__(self, block_name: str = "pg-inkreach-dmp") -> None:
|
|
from prefect_sqlalchemy import SqlAlchemyConnector
|
|
|
|
self._engine = SqlAlchemyConnector.load(block_name).get_engine()
|
|
|
|
def _conn(self):
|
|
"""取连接并预设 lock_timeout(session 级,池化复用的连接同样生效)。"""
|
|
conn = self._engine.raw_connection()
|
|
with conn.cursor() as cur:
|
|
cur.execute(f"SET lock_timeout = '{LOCK_TIMEOUT}'")
|
|
return conn
|
|
|
|
def _retryable_write(self, fn: Callable[[Any], Any]) -> Any:
|
|
"""整事务重试:仅对死锁/等锁超时(DeadlockDetected / LockNotAvailable)重试,其余异常原样抛出。
|
|
|
|
fn(conn) 负责占位锁与 commit;失败时连接已回滚,重试开新连接重跑整个事务。
|
|
"""
|
|
last_err = None
|
|
for attempt in range(WRITE_RETRIES):
|
|
conn = self._conn()
|
|
try:
|
|
result = fn(conn)
|
|
except WRITE_RETRYABLE_ERRORS as err:
|
|
conn.rollback()
|
|
conn.close()
|
|
last_err = err
|
|
if attempt == WRITE_RETRIES - 1:
|
|
raise
|
|
wait = WRITE_RETRY_BACKOFF[attempt]
|
|
print(f"[DB sync] 写库锁冲突({err.__class__.__name__}),{wait}s 后重试"
|
|
f"(第 {attempt + 2}/{WRITE_RETRIES} 次)")
|
|
time.sleep(wait)
|
|
continue
|
|
except Exception:
|
|
conn.rollback()
|
|
conn.close()
|
|
raise
|
|
conn.close()
|
|
return result
|
|
raise last_err # pragma: no cover(循环内必 return/raise)
|
|
|
|
# ---------- 查询辅助 ----------
|
|
|
|
def get_latest_ordered_at(self, system_name: str) -> datetime | None:
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'SELECT MAX(COALESCE("orderedAt","createdAt")) FROM "Orders" WHERE "systemName"=%s',
|
|
(system_name,),
|
|
)
|
|
row = cur.fetchone()
|
|
return row[0] if row and row[0] else None
|
|
|
|
def get_pending_orders(self, system_name: str, exclude_statuses: list[str]) -> list[str]:
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s '
|
|
'AND ("systemStatus" IS NULL OR "systemStatus" <> ALL(%s))',
|
|
(system_name, exclude_statuses),
|
|
)
|
|
rows = cur.fetchall()
|
|
return [r[0] for r in rows]
|
|
|
|
def get_order_count_by_status(self, system_name: str, start: datetime, end: datetime) -> dict[str, int]:
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'SELECT "systemStatus", COUNT(*) FROM "Orders" WHERE "systemName"=%s '
|
|
'AND "orderedAt">=%s AND "orderedAt"<%s GROUP BY "systemStatus"',
|
|
(system_name, start, end),
|
|
)
|
|
rows = cur.fetchall()
|
|
return {r[0]: int(r[1]) for r in rows}
|
|
|
|
def get_local_order_nos(self, system_name: str, start: datetime, end: datetime) -> list[str]:
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s '
|
|
'AND "orderedAt">=%s AND "orderedAt"<%s',
|
|
(system_name, start, end),
|
|
)
|
|
rows = cur.fetchall()
|
|
return [r[0] for r in rows]
|
|
|
|
def get_local_goods_count(self, system_name: str, start: datetime, end: datetime) -> int:
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'SELECT COUNT(*) FROM "Goods" g JOIN "Orders" o '
|
|
'ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" '
|
|
'WHERE o."systemName"=%s AND o."orderedAt">=%s AND o."orderedAt"<%s',
|
|
(system_name, start, end),
|
|
)
|
|
row = cur.fetchone()
|
|
return int(row[0]) if row else 0
|
|
|
|
def get_goods_integrity(self, system_name: str, start: datetime, end: datetime) -> dict[str, int]:
|
|
"""商品数据完整性(与核验同一时间窗,按 orderedAt):
|
|
- orders: 窗口内订单数
|
|
- goods: 窗口内商品行数
|
|
- zeroGoodsOrders: 窗口内没有任何商品行的订单数(可疑缺拉)
|
|
- goodsNoName: 窗口内商品 itemName 为空的行数
|
|
- orphanGoods: 全库没有对应订单的商品行数(脏数据)
|
|
"""
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'SELECT COUNT(*), '
|
|
'COALESCE(SUM(CASE WHEN NOT EXISTS ('
|
|
' SELECT 1 FROM "Goods" g WHERE g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt"'
|
|
') THEN 1 ELSE 0 END), 0) '
|
|
'FROM "Orders" o WHERE o."systemName"=%s AND o."orderedAt">=%s AND o."orderedAt"<%s',
|
|
(system_name, start, end),
|
|
)
|
|
orders, zero_goods = cur.fetchone()
|
|
cur.execute(
|
|
'SELECT COUNT(*), COALESCE(SUM(CASE WHEN "itemName" IS NULL OR "itemName"=\'\' '
|
|
'THEN 1 ELSE 0 END), 0) FROM "Goods" g JOIN "Orders" o '
|
|
'ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" '
|
|
'WHERE o."systemName"=%s AND o."orderedAt">=%s AND o."orderedAt"<%s',
|
|
(system_name, start, end),
|
|
)
|
|
goods, no_name = cur.fetchone()
|
|
cur.execute(
|
|
'SELECT COUNT(*) FROM "Goods" g WHERE NOT EXISTS '
|
|
'(SELECT 1 FROM "Orders" o WHERE o."orderNo"=g."orderNo")',
|
|
)
|
|
orphan = cur.fetchone()[0]
|
|
return {
|
|
"orders": int(orders), "goods": int(goods),
|
|
"zeroGoodsOrders": int(zero_goods), "goodsNoName": int(no_name), "orphanGoods": int(orphan),
|
|
}
|
|
|
|
def ping(self) -> str:
|
|
"""连通性检查,返回 PG 版本号(供监控探活)。"""
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT version()")
|
|
row = cur.fetchone()
|
|
return (row[0] or "").split(",")[0] if row else ""
|
|
|
|
def get_orders_with_100_prefix(self, system_name: str, statuses: list[str]) -> list[tuple[str, datetime]]:
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'SELECT DISTINCT o."orderNo", o."createdAt" FROM "Orders" o '
|
|
'JOIN "Goods" g ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" '
|
|
'WHERE o."systemName"=%s AND o."systemStatus"=ANY(%s) AND g."productionNo" LIKE \'100%%\'',
|
|
(system_name, statuses),
|
|
)
|
|
rows = cur.fetchall()
|
|
return [(r[0], r[1]) for r in rows]
|
|
|
|
# ---------- OCR 面单 enrichment ----------
|
|
|
|
def get_ocr_pending(
|
|
self, limit: int = 0,
|
|
country_regions: list[str] | None = None,
|
|
courier_source: str | None = None,
|
|
days: int = 0,
|
|
) -> list[tuple[str, datetime, str, str | None]]:
|
|
"""查待 OCR 订单(所有系统):(orderNo, createdAt, labelPdfUrl, countryRegion)。
|
|
|
|
条件:有 labelPdfUrl、ocrLayout 未处理(Null)。
|
|
limit=0 表示不限。可选按 countryRegions / courierSource 筛选。
|
|
days>0 时只取近 N 天下单的订单。
|
|
"""
|
|
sql = (
|
|
'SELECT "orderNo","createdAt","labelPdfUrl","countryRegion" '
|
|
'FROM "Orders" '
|
|
'WHERE "labelPdfUrl" IS NOT NULL '
|
|
'AND ("ocrLayout" IS NULL '
|
|
' OR ("ocrLayout" NOT IN (\'ERROR\',\'UNSUPPORTED\') '
|
|
' AND COALESCE("ocrTracking", \'\') = \'\'))'
|
|
)
|
|
params: list = []
|
|
if country_regions:
|
|
sql += ' AND "countryRegion"=ANY(%s)'
|
|
params.append(country_regions)
|
|
if courier_source:
|
|
sql += ' AND "courierSource"=%s'
|
|
params.append(courier_source)
|
|
if days > 0:
|
|
sql += ' AND "orderedAt" >= %s'
|
|
params.append(datetime.now() - timedelta(days=days))
|
|
sql += ' ORDER BY "orderedAt" DESC'
|
|
if limit > 0:
|
|
sql += " LIMIT %s"
|
|
params.append(limit)
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(sql, params)
|
|
return [(r[0], r[1], r[2], r[3]) for r in cur.fetchall()]
|
|
|
|
def get_orders_without_label(self, system_name: str, days: int = 7) -> list[str]:
|
|
"""查近 N 天下单但无 labelPdfUrl 的订单号。"""
|
|
cutoff = datetime.now() - timedelta(days=days)
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'SELECT DISTINCT "orderNo" FROM "Orders" '
|
|
'WHERE "systemName"=%s AND "labelPdfUrl" IS NULL AND "orderedAt" >= %s',
|
|
(system_name, cutoff),
|
|
)
|
|
rows = cur.fetchall()
|
|
return [r[0] for r in rows]
|
|
|
|
def update_ocr_fields(
|
|
self, order_no: str, created_at: datetime, fields: dict[str, str | None]
|
|
) -> None:
|
|
"""更新单个订单的 ocr_* 字段。fields 为 {ocr_xxx: value} 字典。自动盖处理时间戳。"""
|
|
if not fields:
|
|
return
|
|
|
|
def _once(conn) -> None:
|
|
fields_with_ts = {"ocrProcessedAt": datetime.now(), **fields}
|
|
assignments = ", ".join(f'"{k}"=%s' for k in fields_with_ts)
|
|
params = list(fields_with_ts.values()) + [order_no, created_at]
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT pg_advisory_xact_lock(%s)", (DB_WRITE_LOCK_KEY,))
|
|
cur.execute(
|
|
f'''UPDATE "Orders" SET {assignments}
|
|
WHERE "orderNo"=%s AND "createdAt"=%s''',
|
|
params,
|
|
)
|
|
conn.commit()
|
|
|
|
self._retryable_write(_once)
|
|
|
|
def update_ocr_batch(self, updates: list[tuple[str, datetime, dict[str, str | None]]]) -> int:
|
|
"""批量更新 ocr 字段(单连接单事务,与其他写事务互斥)。updates: [(orderNo, createdAt, {field: value})]。
|
|
|
|
返回更新行数。
|
|
"""
|
|
if not updates:
|
|
return 0
|
|
|
|
def _once(conn) -> int:
|
|
rows = 0
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT pg_advisory_xact_lock(%s)", (DB_WRITE_LOCK_KEY,))
|
|
for order_no, created_at, fields in updates:
|
|
if not fields:
|
|
continue
|
|
fields = {"ocrProcessedAt": datetime.now(), **fields}
|
|
assignments = ", ".join(f'"{k}"=%s' for k in fields)
|
|
params = list(fields.values()) + [order_no, created_at]
|
|
cur.execute(
|
|
f'''UPDATE "Orders" SET {assignments}
|
|
WHERE "orderNo"=%s AND "createdAt"=%s''',
|
|
params,
|
|
)
|
|
rows += cur.rowcount
|
|
conn.commit()
|
|
return rows
|
|
|
|
return self._retryable_write(_once)
|
|
|
|
def count_ocr_status(self) -> dict[str, int]:
|
|
"""统计 OCR 处理状态(done/error/unsupported/pending)。"""
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'''SELECT
|
|
COUNT(*) FILTER (WHERE "ocrLayout" IS NOT NULL AND "ocrLayout" NOT IN ('ERROR','UNSUPPORTED')) AS done,
|
|
COUNT(*) FILTER (WHERE "ocrLayout"='ERROR') AS error,
|
|
COUNT(*) FILTER (WHERE "ocrLayout"='UNSUPPORTED') AS unsupported,
|
|
COUNT(*) FILTER (WHERE "ocrLayout" IS NULL AND "labelPdfUrl" IS NOT NULL) AS pending
|
|
FROM "Orders" WHERE "systemName"=%s''',
|
|
("SDS",),
|
|
)
|
|
row = cur.fetchone()
|
|
return {"done": row[0], "error": row[1], "unsupported": row[2], "pending": row[3]} if row else {}
|
|
|
|
def count_ocr_today(self, day_start: datetime) -> dict[str, int]:
|
|
"""今日 OCR 处理量(按 ocrProcessedAt 盖章时间):processed=成功识别, failed=识别失败/不支持。"""
|
|
with self._conn() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'''SELECT
|
|
COUNT(*) FILTER (WHERE "ocrLayout" IS NOT NULL
|
|
AND "ocrLayout" NOT IN ('ERROR','UNSUPPORTED')) AS processed,
|
|
COUNT(*) FILTER (WHERE "ocrLayout" IN ('ERROR','UNSUPPORTED')) AS failed
|
|
FROM "Orders" WHERE "ocrProcessedAt">=%s''',
|
|
(day_start,),
|
|
)
|
|
row = cur.fetchone()
|
|
return {"processed": row[0], "failed": row[1]} if row else {"processed": 0, "failed": 0}
|
|
|
|
# ---------- 主 upsert ----------
|
|
|
|
def sync_orders_and_goods(
|
|
self,
|
|
orders: list[OrderUpsertInput],
|
|
goods: list[GoodsUpsertInput],
|
|
delete_order_nos: list[str] | None = None,
|
|
) -> dict[str, int]:
|
|
result = {
|
|
"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0,
|
|
"goodsUpdated": 0, "goodsRemoved": 0, "ordersDeleted": 0,
|
|
}
|
|
if not orders and not goods and not delete_order_nos:
|
|
print("[DB sync] 无数据可写入,跳过")
|
|
return result
|
|
return self._retryable_write(
|
|
lambda conn: self._sync_orders_and_goods_once(conn, orders, goods, delete_order_nos)
|
|
)
|
|
|
|
def _sync_orders_and_goods_once(
|
|
self,
|
|
conn,
|
|
orders: list[OrderUpsertInput],
|
|
goods: list[GoodsUpsertInput],
|
|
delete_order_nos: list[str] | None,
|
|
) -> dict[str, int]:
|
|
"""单次写库事务:先取共享 advisory 锁与所有写事务串行化,再逐批 upsert。
|
|
|
|
每批(ORDER_CHUNK 个订单)独立提交:持锁窗口从"整个 pending 集"缩到"单批",
|
|
并发写者排队时间以秒计;事务级 advisory 锁随每批 commit 释放,单锁无环,无死锁可能。
|
|
"""
|
|
result = {
|
|
"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0,
|
|
"goodsUpdated": 0, "goodsRemoved": 0, "ordersDeleted": 0,
|
|
}
|
|
try:
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT pg_advisory_xact_lock(%s)", (DB_WRITE_LOCK_KEY,))
|
|
|
|
system_name = orders[0].systemName if orders else None
|
|
all_order_nos = list({o.orderNo for o in orders} | {g.orderNo for g in goods})
|
|
|
|
# 1. 解析已有 createdAt,覆盖到 orders(保证幂等 upsert)
|
|
existing_created: dict[str, datetime] = {}
|
|
for i in range(0, len(all_order_nos), LOOKUP_BATCH):
|
|
chunk = all_order_nos[i : i + LOOKUP_BATCH]
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)'
|
|
+ (f' AND "systemName"=%s' if system_name else ""),
|
|
(chunk, system_name) if system_name else (chunk,),
|
|
)
|
|
for r in cur.fetchall():
|
|
existing_created[r[0]] = r[1]
|
|
for o in orders:
|
|
if o.orderNo in existing_created:
|
|
o.createdAt = existing_created[o.orderNo]
|
|
|
|
# 2. 去重 orders / goods
|
|
seen_o: set[str] = set()
|
|
orders_dedup: list[OrderUpsertInput] = []
|
|
for o in orders:
|
|
key = f'{o.orderNo}|{o.createdAt.isoformat()}'
|
|
if key in seen_o:
|
|
continue
|
|
seen_o.add(key)
|
|
orders_dedup.append(o)
|
|
|
|
resolved_created: dict[str, datetime] = {}
|
|
for o in orders_dedup:
|
|
resolved_created.setdefault(o.orderNo, o.createdAt)
|
|
|
|
seen_g: set[str] = set()
|
|
goods_dedup: list[GoodsUpsertInput] = []
|
|
for g in goods:
|
|
if g.orderNo in resolved_created:
|
|
g.createdAt = resolved_created[g.orderNo]
|
|
key = f'{g.orderNo}|{g.productionNo}'
|
|
if key in seen_g:
|
|
continue
|
|
seen_g.add(key)
|
|
goods_dedup.append(g)
|
|
|
|
goods_by_order: dict[str, list[GoodsUpsertInput]] = {}
|
|
for g in goods_dedup:
|
|
goods_by_order.setdefault(g.orderNo, []).append(g)
|
|
|
|
print(f"[DB sync] 准备写入: {len(orders_dedup)} 订单 / {len(goods_dedup)} 商品"
|
|
+ (f" (系统={system_name})" if system_name else ""))
|
|
|
|
# 3. 分批 upsert orders + 关联 goods(每批提交一次)
|
|
total_order_batches = -(-len(orders_dedup) // ORDER_CHUNK)
|
|
for i in range(0, len(orders_dedup), ORDER_CHUNK):
|
|
chunk = orders_dedup[i : i + ORDER_CHUNK]
|
|
related: list[GoodsUpsertInput] = []
|
|
for o in chunk:
|
|
related.extend(goods_by_order.get(o.orderNo, []))
|
|
batch_no = i // ORDER_CHUNK + 1
|
|
print(f"[DB sync] 订单批次 {batch_no}/{total_order_batches}: {len(chunk)} 订单 / {len(related)} 商品")
|
|
stats = self._upsert_order_batch(conn, chunk, related, system_name)
|
|
for k, v in stats.items():
|
|
result[k] += v
|
|
conn.commit()
|
|
|
|
# 4. 剩余 goods(其 order 不在本次 orders 列表)
|
|
orders_set = {o.orderNo for o in orders_dedup}
|
|
remaining = [g for g in goods_dedup if g.orderNo not in orders_set]
|
|
if remaining:
|
|
c, u = self._upsert_goods_only(conn, remaining, system_name)
|
|
result["goodsCreated"] += c
|
|
result["goodsUpdated"] += u
|
|
|
|
# 5. 删除订单
|
|
if delete_order_nos:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'DELETE FROM "Orders" WHERE "orderNo"=ANY(%s)'
|
|
+ (f' AND "systemName"=%s' if system_name else ""),
|
|
(delete_order_nos, system_name) if system_name else (delete_order_nos,),
|
|
)
|
|
result["ordersDeleted"] = cur.rowcount
|
|
|
|
# 6. 商家占位
|
|
self._ensure_merchants(conn, orders_dedup)
|
|
conn.commit()
|
|
except Exception:
|
|
conn.rollback()
|
|
raise
|
|
print(f"[DB sync] 写入完成: {result}")
|
|
return result
|
|
|
|
def _upsert_order_batch(
|
|
self, conn, orders: list[OrderUpsertInput], goods: list[GoodsUpsertInput], system_name: str | None
|
|
) -> dict[str, int]:
|
|
stats = {"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0, "goodsUpdated": 0, "goodsRemoved": 0}
|
|
with conn.cursor() as cur:
|
|
# 统计已存在 → created/updated
|
|
# 注:必须 cast 成 timestamptz——入参是带时区偏移的 tz-aware datetime,
|
|
# cast 成 timestamp 会丢弃偏移,在会话时区非 +08 时匹配不到既有行,导致 created/updated 统计全错。
|
|
pairs = [(o.orderNo, o.createdAt) for o in orders]
|
|
cur.execute(
|
|
"SELECT t.order_no, t.created_at FROM unnest(%s::text[], %s::timestamptz[]) AS t(order_no,created_at) "
|
|
"JOIN \"Orders\" o ON o.\"orderNo\"=t.order_no AND o.\"createdAt\"=t.created_at",
|
|
_unnest_pairs(pairs),
|
|
)
|
|
existing_set = {(r[0], r[1]) for r in cur.fetchall()}
|
|
updated = sum(1 for o in orders if (o.orderNo, o.createdAt) in existing_set)
|
|
stats["ordersUpdated"] = updated
|
|
stats["ordersCreated"] = len(orders) - updated
|
|
|
|
execute_values(
|
|
cur,
|
|
'''INSERT INTO "Orders" (
|
|
"orderNo","outOrderNo","courierSource","courierName","courierNo","merchantId",
|
|
"countryRegion","factoryName","systemName","systemStatus","platformName","platformStatus",
|
|
"createdAt","orderedAt","confirmedAt","completedAt","shipDeadline",
|
|
"consigneeName","destinationCountry","stateProvince","city",
|
|
"addressLine1","addressLine2","addressLine3","postalCode","storeName","labelPdfUrl"
|
|
) VALUES %s
|
|
ON CONFLICT ("orderNo","createdAt") DO UPDATE SET
|
|
"outOrderNo"=EXCLUDED."outOrderNo","courierSource"=EXCLUDED."courierSource",
|
|
"courierName"=EXCLUDED."courierName","courierNo"=EXCLUDED."courierNo",
|
|
"merchantId"=EXCLUDED."merchantId","countryRegion"=EXCLUDED."countryRegion",
|
|
"factoryName"=EXCLUDED."factoryName","systemName"=EXCLUDED."systemName",
|
|
"systemStatus"=EXCLUDED."systemStatus","platformName"=EXCLUDED."platformName",
|
|
"platformStatus"=EXCLUDED."platformStatus","orderedAt"=EXCLUDED."orderedAt",
|
|
"confirmedAt"=EXCLUDED."confirmedAt","completedAt"=EXCLUDED."completedAt",
|
|
"shipDeadline"=EXCLUDED."shipDeadline",
|
|
"consigneeName"=EXCLUDED."consigneeName",
|
|
"destinationCountry"=EXCLUDED."destinationCountry",
|
|
"stateProvince"=EXCLUDED."stateProvince","city"=EXCLUDED."city",
|
|
"addressLine1"=EXCLUDED."addressLine1","addressLine2"=EXCLUDED."addressLine2",
|
|
"addressLine3"=EXCLUDED."addressLine3","postalCode"=EXCLUDED."postalCode",
|
|
"storeName"=EXCLUDED."storeName","labelPdfUrl"=EXCLUDED."labelPdfUrl"''',
|
|
[(
|
|
o.orderNo, o.outOrderNo, o.courierSource, o.courierName, o.courierNo, o.merchantId,
|
|
o.countryRegion, o.factoryName, o.systemName, o.systemStatus, o.platformName, o.platformStatus,
|
|
o.createdAt, o.orderedAt, o.confirmedAt, o.completedAt, o.shipDeadline,
|
|
o.consigneeName, o.destinationCountry, o.stateProvince, o.city,
|
|
o.addressLine1, o.addressLine2, o.addressLine3, o.postalCode, o.storeName, o.labelPdfUrl,
|
|
) for o in orders],
|
|
page_size=ORDER_CHUNK,
|
|
)
|
|
|
|
# 关联 goods:先删本次不再出现的,再 upsert
|
|
if goods:
|
|
order_nos = list({g.orderNo for g in goods})
|
|
prod_nos = [g.productionNo for g in goods]
|
|
cur.execute(
|
|
'''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s)
|
|
AND ("orderNo","createdAt") IN (
|
|
SELECT "orderNo","createdAt" FROM "Orders"
|
|
WHERE "orderNo"=ANY(%s)''' + (f' AND "systemName"=%s' if system_name else "") + ')',
|
|
(order_nos, prod_nos, order_nos, system_name) if system_name else (order_nos, prod_nos, order_nos),
|
|
)
|
|
stats["goodsRemoved"] += cur.rowcount
|
|
|
|
g_pairs = [(g.orderNo, g.productionNo) for g in goods]
|
|
cur.execute(
|
|
"SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) "
|
|
"JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no",
|
|
_unnest_pairs(g_pairs),
|
|
)
|
|
g_existing = {(r[0], r[1]) for r in cur.fetchall()}
|
|
g_updated = sum(1 for g in goods if (g.orderNo, g.productionNo) in g_existing)
|
|
stats["goodsUpdated"] = g_updated
|
|
stats["goodsCreated"] = len(goods) - g_updated
|
|
|
|
execute_values(
|
|
cur,
|
|
'''INSERT INTO "Goods" (
|
|
"orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor",
|
|
"itemSize","itemStyleNo","quantity","status"
|
|
) VALUES %s
|
|
ON CONFLICT ("orderNo","productionNo") DO UPDATE SET
|
|
"createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo",
|
|
"itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor",
|
|
"itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo",
|
|
"quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''',
|
|
[(
|
|
g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor,
|
|
g.itemSize, g.itemStyleNo, g.quantity, g.status,
|
|
) for g in goods],
|
|
page_size=GOODS_CHUNK,
|
|
)
|
|
return stats
|
|
|
|
def _upsert_goods_only(
|
|
self, conn, goods: list[GoodsUpsertInput], system_name: str | None
|
|
) -> tuple[int, int]:
|
|
# 用库中实际 createdAt 覆盖
|
|
order_nos = list({g.orderNo for g in goods})
|
|
actual: dict[str, datetime] = {}
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)'
|
|
+ (f' AND "systemName"=%s' if system_name else ""),
|
|
(order_nos, system_name) if system_name else (order_nos,),
|
|
)
|
|
for r in cur.fetchall():
|
|
actual[r[0]] = r[1]
|
|
for g in goods:
|
|
if g.orderNo in actual:
|
|
g.createdAt = actual[g.orderNo]
|
|
|
|
created = updated = 0
|
|
for i in range(0, len(goods), GOODS_CHUNK):
|
|
chunk = goods[i : i + GOODS_CHUNK]
|
|
with conn.cursor() as cur:
|
|
c_order = list({g.orderNo for g in chunk})
|
|
c_prod = [g.productionNo for g in chunk]
|
|
cur.execute(
|
|
'''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s)'''
|
|
+ (f''' AND ("orderNo","createdAt") IN (SELECT "orderNo","createdAt" FROM "Orders"
|
|
WHERE "orderNo"=ANY(%s) AND "systemName"=%s)''' if system_name else ""),
|
|
(c_order, c_prod, c_order, system_name) if system_name else (c_order, c_prod),
|
|
)
|
|
g_pairs = [(g.orderNo, g.productionNo) for g in chunk]
|
|
cur.execute(
|
|
"SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) "
|
|
"JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no",
|
|
_unnest_pairs(g_pairs),
|
|
)
|
|
g_existing = {(r[0], r[1]) for r in cur.fetchall()}
|
|
u = sum(1 for g in chunk if (g.orderNo, g.productionNo) in g_existing)
|
|
updated += u
|
|
created += len(chunk) - u
|
|
execute_values(
|
|
cur,
|
|
'''INSERT INTO "Goods" (
|
|
"orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor",
|
|
"itemSize","itemStyleNo","quantity","status"
|
|
) VALUES %s
|
|
ON CONFLICT ("orderNo","productionNo") DO UPDATE SET
|
|
"createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo",
|
|
"itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor",
|
|
"itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo",
|
|
"quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''',
|
|
[(
|
|
g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor,
|
|
g.itemSize, g.itemStyleNo, g.quantity, g.status,
|
|
) for g in chunk],
|
|
page_size=GOODS_CHUNK,
|
|
)
|
|
return created, updated
|
|
|
|
def _ensure_merchants(self, conn, orders: list[OrderUpsertInput]) -> None:
|
|
ids = list({o.merchantId for o in orders if o.merchantId and o.merchantId.strip()})
|
|
if not ids:
|
|
return
|
|
with conn.cursor() as cur:
|
|
execute_values(
|
|
cur,
|
|
'''INSERT INTO "Merchant" ("merchantId","merchantGroup","responsibleName","responsibleId","tags")
|
|
VALUES %s ON CONFLICT ("merchantId") DO NOTHING''',
|
|
[(mid, "", None, None, "") for mid in ids],
|
|
)
|
|
|
|
|
|
def _unnest_pairs(pairs: list[tuple]) -> tuple[list, list]:
|
|
"""构造 psycopg2 unnest 双数组参数:返回 (col_a_list, col_b_list)。
|
|
|
|
配合 SQL:unnest(%s::text[], %s::timestamptz[]) 或 unnest(%s::text[], %s::text[])。
|
|
"""
|
|
return [p[0] for p in pairs], [p[1] for p in pairs]
|