"""PG 同步写入层:Orders / Goods / Merchant 的 upsert + 同步所需的查询。 忠实移植自 inkreach-bi-auth-mng 的 sync/lib/db.service.ts。 - Orders 主键:("orderNo", "createdAt") - Goods 主键:("orderNo", "productionNo") 关键:写入前用库中已有 createdAt 覆盖,避免同一订单因 createdAt 不同产生重复主键行。 """ from __future__ import annotations import time from datetime import datetime, timedelta from typing import Any, Callable from psycopg2 import errors as pg_errors from psycopg2.extras import execute_values from flows.shared.mappers import GoodsUpsertInput, OrderUpsertInput ORDER_CHUNK = 1000 GOODS_CHUNK = 2000 LOOKUP_BATCH = 10000 # 写 Orders/Goods 的互斥:所有写事务先取同一把 advisory 锁(事务级,commit/rollback 即释放)。 # 单锁无环,写事务之间从结构上不可能死锁。1768851058 = "inkr" 的 ASCII 整数表示。 DB_WRITE_LOCK_KEY = 1768851058 LOCK_TIMEOUT = "15s" # 等锁超时 → LockNotAvailable → 交给重试兜底 WRITE_RETRIES = 3 # 死锁/等锁超时的整事务重试次数 WRITE_RETRY_BACKOFF = [1, 3, 7] # 重试退避(秒) WRITE_RETRYABLE_ERRORS = (pg_errors.DeadlockDetected, pg_errors.LockNotAvailable) class SyncDB: def __init__(self, block_name: str = "pg-inkreach-dmp") -> None: from prefect_sqlalchemy import SqlAlchemyConnector self._engine = SqlAlchemyConnector.load(block_name).get_engine() def _conn(self): """取连接并预设 lock_timeout(session 级,池化复用的连接同样生效)。""" conn = self._engine.raw_connection() with conn.cursor() as cur: cur.execute(f"SET lock_timeout = '{LOCK_TIMEOUT}'") return conn def _retryable_write(self, fn: Callable[[Any], Any]) -> Any: """整事务重试:仅对死锁/等锁超时(DeadlockDetected / LockNotAvailable)重试,其余异常原样抛出。 fn(conn) 负责占位锁与 commit;失败时连接已回滚,重试开新连接重跑整个事务。 """ last_err = None for attempt in range(WRITE_RETRIES): conn = self._conn() try: result = fn(conn) except WRITE_RETRYABLE_ERRORS as err: conn.rollback() conn.close() last_err = err if attempt == WRITE_RETRIES - 1: raise wait = WRITE_RETRY_BACKOFF[attempt] print(f"[DB sync] 写库锁冲突({err.__class__.__name__}),{wait}s 后重试" f"(第 {attempt + 2}/{WRITE_RETRIES} 次)") time.sleep(wait) continue except Exception: conn.rollback() conn.close() raise conn.close() return result raise last_err # pragma: no cover(循环内必 return/raise) # ---------- 查询辅助 ---------- def get_latest_ordered_at(self, system_name: str) -> datetime | None: with self._conn() as conn: with conn.cursor() as cur: cur.execute( 'SELECT MAX(COALESCE("orderedAt","createdAt")) FROM "Orders" WHERE "systemName"=%s', (system_name,), ) row = cur.fetchone() return row[0] if row and row[0] else None def get_pending_orders(self, system_name: str, exclude_statuses: list[str]) -> list[str]: with self._conn() as conn: with conn.cursor() as cur: cur.execute( 'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s ' 'AND ("systemStatus" IS NULL OR "systemStatus" <> ALL(%s))', (system_name, exclude_statuses), ) rows = cur.fetchall() return [r[0] for r in rows] def get_order_count_by_status(self, system_name: str, start: datetime, end: datetime) -> dict[str, int]: with self._conn() as conn: with conn.cursor() as cur: cur.execute( 'SELECT "systemStatus", COUNT(*) FROM "Orders" WHERE "systemName"=%s ' 'AND "orderedAt">=%s AND "orderedAt"<%s GROUP BY "systemStatus"', (system_name, start, end), ) rows = cur.fetchall() return {r[0]: int(r[1]) for r in rows} def get_local_order_nos(self, system_name: str, start: datetime, end: datetime) -> list[str]: with self._conn() as conn: with conn.cursor() as cur: cur.execute( 'SELECT DISTINCT "orderNo" FROM "Orders" WHERE "systemName"=%s ' 'AND "orderedAt">=%s AND "orderedAt"<%s', (system_name, start, end), ) rows = cur.fetchall() return [r[0] for r in rows] def get_local_goods_count(self, system_name: str, start: datetime, end: datetime) -> int: with self._conn() as conn: with conn.cursor() as cur: cur.execute( 'SELECT COUNT(*) FROM "Goods" g JOIN "Orders" o ' 'ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" ' 'WHERE o."systemName"=%s AND o."orderedAt">=%s AND o."orderedAt"<%s', (system_name, start, end), ) row = cur.fetchone() return int(row[0]) if row else 0 def get_goods_integrity(self, system_name: str, start: datetime, end: datetime) -> dict[str, int]: """商品数据完整性(与核验同一时间窗,按 orderedAt): - orders: 窗口内订单数 - goods: 窗口内商品行数 - zeroGoodsOrders: 窗口内没有任何商品行的订单数(可疑缺拉) - goodsNoName: 窗口内商品 itemName 为空的行数 - orphanGoods: 全库没有对应订单的商品行数(脏数据) """ with self._conn() as conn: with conn.cursor() as cur: cur.execute( 'SELECT COUNT(*), ' 'COALESCE(SUM(CASE WHEN NOT EXISTS (' ' SELECT 1 FROM "Goods" g WHERE g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt"' ') THEN 1 ELSE 0 END), 0) ' 'FROM "Orders" o WHERE o."systemName"=%s AND o."orderedAt">=%s AND o."orderedAt"<%s', (system_name, start, end), ) orders, zero_goods = cur.fetchone() cur.execute( 'SELECT COUNT(*), COALESCE(SUM(CASE WHEN "itemName" IS NULL OR "itemName"=\'\' ' 'THEN 1 ELSE 0 END), 0) FROM "Goods" g JOIN "Orders" o ' 'ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" ' 'WHERE o."systemName"=%s AND o."orderedAt">=%s AND o."orderedAt"<%s', (system_name, start, end), ) goods, no_name = cur.fetchone() cur.execute( 'SELECT COUNT(*) FROM "Goods" g WHERE NOT EXISTS ' '(SELECT 1 FROM "Orders" o WHERE o."orderNo"=g."orderNo")', ) orphan = cur.fetchone()[0] return { "orders": int(orders), "goods": int(goods), "zeroGoodsOrders": int(zero_goods), "goodsNoName": int(no_name), "orphanGoods": int(orphan), } def ping(self) -> str: """连通性检查,返回 PG 版本号(供监控探活)。""" with self._conn() as conn: with conn.cursor() as cur: cur.execute("SELECT version()") row = cur.fetchone() return (row[0] or "").split(",")[0] if row else "" def get_orders_with_100_prefix(self, system_name: str, statuses: list[str]) -> list[tuple[str, datetime]]: with self._conn() as conn: with conn.cursor() as cur: cur.execute( 'SELECT DISTINCT o."orderNo", o."createdAt" FROM "Orders" o ' 'JOIN "Goods" g ON g."orderNo"=o."orderNo" AND g."createdAt"=o."createdAt" ' 'WHERE o."systemName"=%s AND o."systemStatus"=ANY(%s) AND g."productionNo" LIKE \'100%%\'', (system_name, statuses), ) rows = cur.fetchall() return [(r[0], r[1]) for r in rows] # ---------- OCR 面单 enrichment ---------- def get_ocr_pending( self, limit: int = 0, country_regions: list[str] | None = None, courier_source: str | None = None, days: int = 0, ) -> list[tuple[str, datetime, str, str | None]]: """查待 OCR 订单(所有系统):(orderNo, createdAt, labelPdfUrl, countryRegion)。 条件:有 labelPdfUrl、ocrLayout 未处理(Null)。 limit=0 表示不限。可选按 countryRegions / courierSource 筛选。 days>0 时只取近 N 天下单的订单。 """ sql = ( 'SELECT "orderNo","createdAt","labelPdfUrl","countryRegion" ' 'FROM "Orders" ' 'WHERE "labelPdfUrl" IS NOT NULL ' 'AND ("ocrLayout" IS NULL ' ' OR ("ocrLayout" NOT IN (\'ERROR\',\'UNSUPPORTED\') ' ' AND COALESCE("ocrTracking", \'\') = \'\'))' ) params: list = [] if country_regions: sql += ' AND "countryRegion"=ANY(%s)' params.append(country_regions) if courier_source: sql += ' AND "courierSource"=%s' params.append(courier_source) if days > 0: sql += ' AND "orderedAt" >= %s' params.append(datetime.now() - timedelta(days=days)) sql += ' ORDER BY "orderedAt" DESC' if limit > 0: sql += " LIMIT %s" params.append(limit) with self._conn() as conn: with conn.cursor() as cur: cur.execute(sql, params) return [(r[0], r[1], r[2], r[3]) for r in cur.fetchall()] def get_orders_without_label(self, system_name: str, days: int = 7) -> list[str]: """查近 N 天下单但无 labelPdfUrl 的订单号。""" cutoff = datetime.now() - timedelta(days=days) with self._conn() as conn: with conn.cursor() as cur: cur.execute( 'SELECT DISTINCT "orderNo" FROM "Orders" ' 'WHERE "systemName"=%s AND "labelPdfUrl" IS NULL AND "orderedAt" >= %s', (system_name, cutoff), ) rows = cur.fetchall() return [r[0] for r in rows] def update_ocr_fields( self, order_no: str, created_at: datetime, fields: dict[str, str | None] ) -> None: """更新单个订单的 ocr_* 字段。fields 为 {ocr_xxx: value} 字典。自动盖处理时间戳。""" if not fields: return def _once(conn) -> None: fields_with_ts = {"ocrProcessedAt": datetime.now(), **fields} assignments = ", ".join(f'"{k}"=%s' for k in fields_with_ts) params = list(fields_with_ts.values()) + [order_no, created_at] with conn.cursor() as cur: cur.execute("SELECT pg_advisory_xact_lock(%s)", (DB_WRITE_LOCK_KEY,)) cur.execute( f'''UPDATE "Orders" SET {assignments} WHERE "orderNo"=%s AND "createdAt"=%s''', params, ) conn.commit() self._retryable_write(_once) def update_ocr_batch(self, updates: list[tuple[str, datetime, dict[str, str | None]]]) -> int: """批量更新 ocr 字段(单连接单事务,与其他写事务互斥)。updates: [(orderNo, createdAt, {field: value})]。 返回更新行数。 """ if not updates: return 0 def _once(conn) -> int: rows = 0 with conn.cursor() as cur: cur.execute("SELECT pg_advisory_xact_lock(%s)", (DB_WRITE_LOCK_KEY,)) for order_no, created_at, fields in updates: if not fields: continue fields = {"ocrProcessedAt": datetime.now(), **fields} assignments = ", ".join(f'"{k}"=%s' for k in fields) params = list(fields.values()) + [order_no, created_at] cur.execute( f'''UPDATE "Orders" SET {assignments} WHERE "orderNo"=%s AND "createdAt"=%s''', params, ) rows += cur.rowcount conn.commit() return rows return self._retryable_write(_once) def count_ocr_status(self) -> dict[str, int]: """统计 OCR 处理状态(done/error/unsupported/pending)。""" with self._conn() as conn: with conn.cursor() as cur: cur.execute( '''SELECT COUNT(*) FILTER (WHERE "ocrLayout" IS NOT NULL AND "ocrLayout" NOT IN ('ERROR','UNSUPPORTED')) AS done, COUNT(*) FILTER (WHERE "ocrLayout"='ERROR') AS error, COUNT(*) FILTER (WHERE "ocrLayout"='UNSUPPORTED') AS unsupported, COUNT(*) FILTER (WHERE "ocrLayout" IS NULL AND "labelPdfUrl" IS NOT NULL) AS pending FROM "Orders" WHERE "systemName"=%s''', ("SDS",), ) row = cur.fetchone() return {"done": row[0], "error": row[1], "unsupported": row[2], "pending": row[3]} if row else {} def count_ocr_today(self, day_start: datetime) -> dict[str, int]: """今日 OCR 处理量(按 ocrProcessedAt 盖章时间):processed=成功识别, failed=识别失败/不支持。""" with self._conn() as conn: with conn.cursor() as cur: cur.execute( '''SELECT COUNT(*) FILTER (WHERE "ocrLayout" IS NOT NULL AND "ocrLayout" NOT IN ('ERROR','UNSUPPORTED')) AS processed, COUNT(*) FILTER (WHERE "ocrLayout" IN ('ERROR','UNSUPPORTED')) AS failed FROM "Orders" WHERE "ocrProcessedAt">=%s''', (day_start,), ) row = cur.fetchone() return {"processed": row[0], "failed": row[1]} if row else {"processed": 0, "failed": 0} # ---------- 主 upsert ---------- def sync_orders_and_goods( self, orders: list[OrderUpsertInput], goods: list[GoodsUpsertInput], delete_order_nos: list[str] | None = None, ) -> dict[str, int]: result = { "ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0, "goodsUpdated": 0, "goodsRemoved": 0, "ordersDeleted": 0, } if not orders and not goods and not delete_order_nos: print("[DB sync] 无数据可写入,跳过") return result return self._retryable_write( lambda conn: self._sync_orders_and_goods_once(conn, orders, goods, delete_order_nos) ) def _sync_orders_and_goods_once( self, conn, orders: list[OrderUpsertInput], goods: list[GoodsUpsertInput], delete_order_nos: list[str] | None, ) -> dict[str, int]: """单次写库事务:先取共享 advisory 锁与所有写事务串行化,再逐批 upsert。 每批(ORDER_CHUNK 个订单)独立提交:持锁窗口从"整个 pending 集"缩到"单批", 并发写者排队时间以秒计;事务级 advisory 锁随每批 commit 释放,单锁无环,无死锁可能。 """ result = { "ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0, "goodsUpdated": 0, "goodsRemoved": 0, "ordersDeleted": 0, } try: with conn.cursor() as cur: cur.execute("SELECT pg_advisory_xact_lock(%s)", (DB_WRITE_LOCK_KEY,)) system_name = orders[0].systemName if orders else None all_order_nos = list({o.orderNo for o in orders} | {g.orderNo for g in goods}) # 1. 解析已有 createdAt,覆盖到 orders(保证幂等 upsert) existing_created: dict[str, datetime] = {} for i in range(0, len(all_order_nos), LOOKUP_BATCH): chunk = all_order_nos[i : i + LOOKUP_BATCH] with conn.cursor() as cur: cur.execute( 'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)' + (f' AND "systemName"=%s' if system_name else ""), (chunk, system_name) if system_name else (chunk,), ) for r in cur.fetchall(): existing_created[r[0]] = r[1] for o in orders: if o.orderNo in existing_created: o.createdAt = existing_created[o.orderNo] # 2. 去重 orders / goods seen_o: set[str] = set() orders_dedup: list[OrderUpsertInput] = [] for o in orders: key = f'{o.orderNo}|{o.createdAt.isoformat()}' if key in seen_o: continue seen_o.add(key) orders_dedup.append(o) resolved_created: dict[str, datetime] = {} for o in orders_dedup: resolved_created.setdefault(o.orderNo, o.createdAt) seen_g: set[str] = set() goods_dedup: list[GoodsUpsertInput] = [] for g in goods: if g.orderNo in resolved_created: g.createdAt = resolved_created[g.orderNo] key = f'{g.orderNo}|{g.productionNo}' if key in seen_g: continue seen_g.add(key) goods_dedup.append(g) goods_by_order: dict[str, list[GoodsUpsertInput]] = {} for g in goods_dedup: goods_by_order.setdefault(g.orderNo, []).append(g) print(f"[DB sync] 准备写入: {len(orders_dedup)} 订单 / {len(goods_dedup)} 商品" + (f" (系统={system_name})" if system_name else "")) # 3. 分批 upsert orders + 关联 goods(每批提交一次) total_order_batches = -(-len(orders_dedup) // ORDER_CHUNK) for i in range(0, len(orders_dedup), ORDER_CHUNK): chunk = orders_dedup[i : i + ORDER_CHUNK] related: list[GoodsUpsertInput] = [] for o in chunk: related.extend(goods_by_order.get(o.orderNo, [])) batch_no = i // ORDER_CHUNK + 1 print(f"[DB sync] 订单批次 {batch_no}/{total_order_batches}: {len(chunk)} 订单 / {len(related)} 商品") stats = self._upsert_order_batch(conn, chunk, related, system_name) for k, v in stats.items(): result[k] += v conn.commit() # 4. 剩余 goods(其 order 不在本次 orders 列表) orders_set = {o.orderNo for o in orders_dedup} remaining = [g for g in goods_dedup if g.orderNo not in orders_set] if remaining: c, u = self._upsert_goods_only(conn, remaining, system_name) result["goodsCreated"] += c result["goodsUpdated"] += u # 5. 删除订单 if delete_order_nos: with conn.cursor() as cur: cur.execute( 'DELETE FROM "Orders" WHERE "orderNo"=ANY(%s)' + (f' AND "systemName"=%s' if system_name else ""), (delete_order_nos, system_name) if system_name else (delete_order_nos,), ) result["ordersDeleted"] = cur.rowcount # 6. 商家占位 self._ensure_merchants(conn, orders_dedup) conn.commit() except Exception: conn.rollback() raise print(f"[DB sync] 写入完成: {result}") return result def _upsert_order_batch( self, conn, orders: list[OrderUpsertInput], goods: list[GoodsUpsertInput], system_name: str | None ) -> dict[str, int]: stats = {"ordersCreated": 0, "ordersUpdated": 0, "goodsCreated": 0, "goodsUpdated": 0, "goodsRemoved": 0} with conn.cursor() as cur: # 统计已存在 → created/updated pairs = [(o.orderNo, o.createdAt) for o in orders] cur.execute( "SELECT t.order_no, t.created_at FROM unnest(%s::text[], %s::timestamp[]) AS t(order_no,created_at) " "JOIN \"Orders\" o ON o.\"orderNo\"=t.order_no AND o.\"createdAt\"=t.created_at", _unnest_pairs(pairs), ) existing_set = {(r[0], r[1]) for r in cur.fetchall()} updated = sum(1 for o in orders if (o.orderNo, o.createdAt) in existing_set) stats["ordersUpdated"] = updated stats["ordersCreated"] = len(orders) - updated execute_values( cur, '''INSERT INTO "Orders" ( "orderNo","outOrderNo","courierSource","courierName","courierNo","merchantId", "countryRegion","factoryName","systemName","systemStatus","platformName","platformStatus", "createdAt","orderedAt","confirmedAt","completedAt","shipDeadline", "consigneeName","destinationCountry","stateProvince","city", "addressLine1","addressLine2","addressLine3","postalCode","storeName","labelPdfUrl" ) VALUES %s ON CONFLICT ("orderNo","createdAt") DO UPDATE SET "outOrderNo"=EXCLUDED."outOrderNo","courierSource"=EXCLUDED."courierSource", "courierName"=EXCLUDED."courierName","courierNo"=EXCLUDED."courierNo", "merchantId"=EXCLUDED."merchantId","countryRegion"=EXCLUDED."countryRegion", "factoryName"=EXCLUDED."factoryName","systemName"=EXCLUDED."systemName", "systemStatus"=EXCLUDED."systemStatus","platformName"=EXCLUDED."platformName", "platformStatus"=EXCLUDED."platformStatus","orderedAt"=EXCLUDED."orderedAt", "confirmedAt"=EXCLUDED."confirmedAt","completedAt"=EXCLUDED."completedAt", "shipDeadline"=EXCLUDED."shipDeadline", "consigneeName"=EXCLUDED."consigneeName", "destinationCountry"=EXCLUDED."destinationCountry", "stateProvince"=EXCLUDED."stateProvince","city"=EXCLUDED."city", "addressLine1"=EXCLUDED."addressLine1","addressLine2"=EXCLUDED."addressLine2", "addressLine3"=EXCLUDED."addressLine3","postalCode"=EXCLUDED."postalCode", "storeName"=EXCLUDED."storeName","labelPdfUrl"=EXCLUDED."labelPdfUrl"''', [( o.orderNo, o.outOrderNo, o.courierSource, o.courierName, o.courierNo, o.merchantId, o.countryRegion, o.factoryName, o.systemName, o.systemStatus, o.platformName, o.platformStatus, o.createdAt, o.orderedAt, o.confirmedAt, o.completedAt, o.shipDeadline, o.consigneeName, o.destinationCountry, o.stateProvince, o.city, o.addressLine1, o.addressLine2, o.addressLine3, o.postalCode, o.storeName, o.labelPdfUrl, ) for o in orders], page_size=ORDER_CHUNK, ) # 关联 goods:先删本次不再出现的,再 upsert if goods: order_nos = list({g.orderNo for g in goods}) prod_nos = [g.productionNo for g in goods] cur.execute( '''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s) AND ("orderNo","createdAt") IN ( SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)''' + (f' AND "systemName"=%s' if system_name else "") + ')', (order_nos, prod_nos, order_nos, system_name) if system_name else (order_nos, prod_nos, order_nos), ) stats["goodsRemoved"] += cur.rowcount g_pairs = [(g.orderNo, g.productionNo) for g in goods] cur.execute( "SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) " "JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no", _unnest_pairs(g_pairs), ) g_existing = {(r[0], r[1]) for r in cur.fetchall()} g_updated = sum(1 for g in goods if (g.orderNo, g.productionNo) in g_existing) stats["goodsUpdated"] = g_updated stats["goodsCreated"] = len(goods) - g_updated execute_values( cur, '''INSERT INTO "Goods" ( "orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor", "itemSize","itemStyleNo","quantity","status" ) VALUES %s ON CONFLICT ("orderNo","productionNo") DO UPDATE SET "createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo", "itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor", "itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo", "quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''', [( g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor, g.itemSize, g.itemStyleNo, g.quantity, g.status, ) for g in goods], page_size=GOODS_CHUNK, ) return stats def _upsert_goods_only( self, conn, goods: list[GoodsUpsertInput], system_name: str | None ) -> tuple[int, int]: # 用库中实际 createdAt 覆盖 order_nos = list({g.orderNo for g in goods}) actual: dict[str, datetime] = {} with conn.cursor() as cur: cur.execute( 'SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s)' + (f' AND "systemName"=%s' if system_name else ""), (order_nos, system_name) if system_name else (order_nos,), ) for r in cur.fetchall(): actual[r[0]] = r[1] for g in goods: if g.orderNo in actual: g.createdAt = actual[g.orderNo] created = updated = 0 for i in range(0, len(goods), GOODS_CHUNK): chunk = goods[i : i + GOODS_CHUNK] with conn.cursor() as cur: c_order = list({g.orderNo for g in chunk}) c_prod = [g.productionNo for g in chunk] cur.execute( '''DELETE FROM "Goods" WHERE "orderNo"=ANY(%s) AND "productionNo" <> ALL(%s)''' + (f''' AND ("orderNo","createdAt") IN (SELECT "orderNo","createdAt" FROM "Orders" WHERE "orderNo"=ANY(%s) AND "systemName"=%s)''' if system_name else ""), (c_order, c_prod, c_order, system_name) if system_name else (c_order, c_prod), ) g_pairs = [(g.orderNo, g.productionNo) for g in chunk] cur.execute( "SELECT t.order_no,t.production_no FROM unnest(%s::text[], %s::text[]) AS t(order_no,production_no) " "JOIN \"Goods\" g ON g.\"orderNo\"=t.order_no AND g.\"productionNo\"=t.production_no", _unnest_pairs(g_pairs), ) g_existing = {(r[0], r[1]) for r in cur.fetchall()} u = sum(1 for g in chunk if (g.orderNo, g.productionNo) in g_existing) updated += u created += len(chunk) - u execute_values( cur, '''INSERT INTO "Goods" ( "orderNo","productionNo","createdAt","outOrderNo","itemName","itemColor", "itemSize","itemStyleNo","quantity","status" ) VALUES %s ON CONFLICT ("orderNo","productionNo") DO UPDATE SET "createdAt"=EXCLUDED."createdAt","outOrderNo"=EXCLUDED."outOrderNo", "itemName"=EXCLUDED."itemName","itemColor"=EXCLUDED."itemColor", "itemSize"=EXCLUDED."itemSize","itemStyleNo"=EXCLUDED."itemStyleNo", "quantity"=EXCLUDED."quantity","status"=EXCLUDED."status"''', [( g.orderNo, g.productionNo, g.createdAt, g.outOrderNo, g.itemName, g.itemColor, g.itemSize, g.itemStyleNo, g.quantity, g.status, ) for g in chunk], page_size=GOODS_CHUNK, ) return created, updated def _ensure_merchants(self, conn, orders: list[OrderUpsertInput]) -> None: ids = list({o.merchantId for o in orders if o.merchantId and o.merchantId.strip()}) if not ids: return with conn.cursor() as cur: execute_values( cur, '''INSERT INTO "Merchant" ("merchantId","merchantGroup","responsibleName","responsibleId","tags") VALUES %s ON CONFLICT ("merchantId") DO NOTHING''', [(mid, "", None, None, "") for mid in ids], ) def _unnest_pairs(pairs: list[tuple]) -> tuple[list, list]: """构造 psycopg2 unnest 双数组参数:返回 (col_a_list, col_b_list)。 配合 SQL:unnest(%s::text[], %s::timestamp[]) 或 unnest(%s::text[], %s::text[])。 """ return [p[0] for p in pairs], [p[1] for p in pairs]