Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c030eca178 | ||
|
|
c6be5bacf6 | ||
|
|
2259bce7c4 | ||
|
|
10d3c5f278 | ||
|
|
6007b2c34b | ||
|
|
d8e4adb0b5 | ||
|
|
f37ef24cf0 | ||
|
|
1393b2a86b | ||
|
|
5aaddbbf7c |
@@ -12,7 +12,10 @@ jobs:
|
||||
- name: 检出代码
|
||||
run: |
|
||||
REF="${{ github.event.pull_request.head.ref || github.ref_name }}"
|
||||
git clone --depth 1 --branch "$REF" "${{ github.server_url }}/${{ github.repository }}.git" .
|
||||
git clone --depth 1 "${{ github.server_url }}/${{ github.repository }}.git" .
|
||||
# PR head 可能是完整 ref(AGit 建的 PR 为 refs/pull/N/head),clone --branch 只认分支名,改 fetch 后 checkout
|
||||
git fetch --depth 1 origin "$REF"
|
||||
git checkout --detach FETCH_HEAD
|
||||
- name: 安装 uv
|
||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
- name: 安装依赖
|
||||
|
||||
+3
-1
@@ -37,14 +37,16 @@ class OcrResult(BaseModel):
|
||||
|
||||
|
||||
# OCR 服务已有分类器模型的国家(版面见 api.md §4)
|
||||
SUPPORTED_COUNTRIES = {"US", "UK", "DE", "MX", "SA"}
|
||||
SUPPORTED_COUNTRIES = {"US", "UK", "DE", "MX", "SA", "CA", "ES", "IT"}
|
||||
|
||||
# ISO 国家代码 / 中文国名 → OCR 国家代码
|
||||
_COUNTRY_MAP: dict[str, str] = {
|
||||
# 代码
|
||||
"GB": "UK", "UK": "UK", "US": "US", "DE": "DE", "MX": "MX", "SA": "SA",
|
||||
"CA": "CA", "ES": "ES", "IT": "IT",
|
||||
# 中文名
|
||||
"英国": "UK", "美国": "US", "德国": "DE", "墨西哥": "MX", "沙特": "SA",
|
||||
"加拿大": "CA", "西班牙": "ES", "意大利": "IT",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -15,6 +15,9 @@ services:
|
||||
prefect-worker:
|
||||
build: .
|
||||
container_name: prefect-worker
|
||||
# init 进程(tini)回收孤儿/僵尸:pull 步骤的 git 子进程被杀后没人 wait 会变僵尸,
|
||||
# 堆满 cgroup PID 配额后容器内无法再 fork(2026-09-17 事故根因)
|
||||
init: true
|
||||
environment:
|
||||
PREFECT_API_URL: http://prefect-server:4200/api
|
||||
TZ: Asia/Shanghai
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
-- 2026-09-28 Goods.status 约束新增允许 '暂停'
|
||||
-- 背景:RIIN 商品级 goodsStatus 复用平台状态枚举,存在 PAUSE(暂停),
|
||||
-- 原约束未包含该值,导致映射层只能落 NULL(chk_goods_status 拒绝其余新值)。
|
||||
-- 配套:mappers.py 的 RIIN_GOODS_STATUS_MAP 补 COMPLETE/PAUSE。
|
||||
-- 执行方式(两步,避免阻塞 ETL 写入;VALIDATE 只锁 SHARE UPDATE EXCLUSIVE):
|
||||
ALTER TABLE "Goods" DROP CONSTRAINT chk_goods_status;
|
||||
ALTER TABLE "Goods" ADD CONSTRAINT chk_goods_status CHECK (
|
||||
status IS NULL OR status = ANY (ARRAY[
|
||||
'备货中', '已删除', '已取消', '已完成', '待付款', '待确认',
|
||||
'搁置中', '待发货', '已发货', '退款中', '暂停'
|
||||
])
|
||||
) NOT VALID;
|
||||
ALTER TABLE "Goods" VALIDATE CONSTRAINT chk_goods_status;
|
||||
+28
-11
@@ -7,6 +7,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import httpx
|
||||
from prefect import flow, task
|
||||
@@ -16,6 +17,8 @@ from api.ocr import OcrClient, country_to_ocr_code
|
||||
from flows.shared import SyncDB, build_stats_rows, emit_sync_summary
|
||||
|
||||
BATCH_SIZE = 200
|
||||
BATCH_RETRIES = 3 # 批量请求级重试(OCR 服务偶发秒级重启)
|
||||
BATCH_RETRY_BACKOFF = [5, 15] # 重试间隔(秒)
|
||||
|
||||
_OCR_FIELD_MAP = [
|
||||
("layout", "ocrLayout"), ("courier", "ocrCourier"), ("tracking", "ocrTracking"),
|
||||
@@ -63,7 +66,7 @@ def _has_running_peer() -> bool:
|
||||
def process_all(pending: list) -> dict[str, int]:
|
||||
db = SyncDB()
|
||||
ocr = OcrClient()
|
||||
stats = {"success": 0, "error": 0, "unsupported": 0, "batches": 0}
|
||||
stats = {"success": 0, "error": 0, "unsupported": 0, "batches": 0, "deferred": 0}
|
||||
try:
|
||||
# 1. 按国家分组;不支持的国家直接标 UNSUPPORTED
|
||||
by_country: dict[str, list[tuple[str, object, str]]] = {}
|
||||
@@ -89,12 +92,24 @@ def process_all(pending: list) -> dict[str, int]:
|
||||
urls = [{"url": url, "filename": no} for no, _, url in batch]
|
||||
batch_no = i // BATCH_SIZE + 1
|
||||
total_batches = -(-total // BATCH_SIZE)
|
||||
try:
|
||||
results = ocr.recognize_batch(urls, country_code)
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[OCR] {country_code} batch {batch_no}/{total_batches} 请求失败: {str(err)[:100]}")
|
||||
db.update_ocr_batch([(no, created_map[no], {"ocrLayout": "ERROR"}) for no in created_map])
|
||||
stats["error"] += len(batch)
|
||||
results = None
|
||||
for attempt in range(BATCH_RETRIES):
|
||||
try:
|
||||
results = ocr.recognize_batch(urls, country_code)
|
||||
break
|
||||
except Exception as err: # noqa: BLE001
|
||||
if attempt < BATCH_RETRIES - 1:
|
||||
wait = BATCH_RETRY_BACKOFF[min(attempt, len(BATCH_RETRY_BACKOFF) - 1)]
|
||||
print(f"[OCR] {country_code} batch {batch_no}/{total_batches} 请求失败"
|
||||
f"({str(err)[:80]}),{wait}s 后重试({attempt + 2}/{BATCH_RETRIES})")
|
||||
time.sleep(wait)
|
||||
else:
|
||||
# OCR 服务端短暂不可用(容器秒级重启类故障):订单保持未处理,
|
||||
# 留给下个小时的 run 自动重试,不落 ERROR(那会是永久终态)。
|
||||
print(f"[OCR] {country_code} batch {batch_no}/{total_batches} "
|
||||
f"重试 {BATCH_RETRIES} 次仍失败,留待下次 run: {str(err)[:100]}")
|
||||
if results is None:
|
||||
stats["deferred"] += len(batch)
|
||||
continue
|
||||
|
||||
updates = []
|
||||
@@ -122,12 +137,12 @@ def label_ocr(
|
||||
courier_source: str | None = None,
|
||||
days: int = 7,
|
||||
) -> dict:
|
||||
"""面单 OCR enrichment。默认处理英国+美国,近 7 天下单的订单。
|
||||
"""面单 OCR enrichment。默认处理全部已支持国家(见 api.ocr.models.SUPPORTED_COUNTRIES),近 7 天下单的订单。
|
||||
|
||||
手动跑其他国家: label_ocr(country_regions=['德国'])
|
||||
手动跑指定国家: label_ocr(country_regions=['德国'])
|
||||
"""
|
||||
if country_regions is None:
|
||||
country_regions = ['英国', '美国', '墨西哥', '德国', '沙特']
|
||||
country_regions = ['英国', '美国', '墨西哥', '德国', '沙特', '加拿大', '西班牙', '意大利']
|
||||
if _has_running_peer():
|
||||
print("已有 label-ocr 在运行,跳过本次")
|
||||
return {"skipped": True}
|
||||
@@ -152,11 +167,13 @@ def label_ocr(
|
||||
after = db.count_ocr_status()
|
||||
total_processed = stats["success"] + stats["error"] + stats["unsupported"]
|
||||
print(f"完成{scope}: 处理 {total_processed} 单 / {stats['batches']} 批 | "
|
||||
f"成功 {stats['success']} 失败 {stats['error']} 不支持 {stats['unsupported']}")
|
||||
f"成功 {stats['success']} 失败 {stats['error']} 不支持 {stats['unsupported']} "
|
||||
f"遗留待重试 {stats['deferred']}")
|
||||
print(f"累计: {after}")
|
||||
emit_sync_summary(f"面单 OCR{scope}", build_stats_rows("ALL", {}, [
|
||||
("本批处理", total_processed), ("成功", stats["success"]),
|
||||
("失败", stats["error"]), ("不支持", stats["unsupported"]),
|
||||
("遗留待重试", stats["deferred"]),
|
||||
("累计已完成", after.get("done", 0)), ("累计待处理", after.get("pending", 0)),
|
||||
]))
|
||||
return {"processed": total_processed, **stats, **after}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""平台订单 platform 字段大写同步:把 platform_order_data.platform 统一转为大写。
|
||||
|
||||
优化自 platform_order_upper_sync.py,对应 platform-order-upper-sync。
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import psycopg2
|
||||
from prefect import flow
|
||||
|
||||
# 数据库连接参数
|
||||
conn_params = {
|
||||
"host": "192.168.124.92",
|
||||
"port": 5432,
|
||||
"user": "postgres",
|
||||
"password": "Inkreach123456.",
|
||||
"database": "smt",
|
||||
}
|
||||
|
||||
|
||||
def log_msg(msg, level="info"):
|
||||
ts = time.strftime('%H:%M:%S')
|
||||
tag = {"success": "[OK]", "error": "[ERR]"}.get(level, "")
|
||||
print(f"[{ts}] {tag} {msg}")
|
||||
|
||||
|
||||
def update_platform_upper() -> int:
|
||||
conn = None
|
||||
cur = None
|
||||
try:
|
||||
conn = psycopg2.connect(**conn_params)
|
||||
cur = conn.cursor()
|
||||
|
||||
# 仅更新需要变更的行,避免每次都全表写入
|
||||
update_sql = """
|
||||
UPDATE platform_order_data
|
||||
SET platform = UPPER(platform)
|
||||
WHERE platform IS NOT NULL
|
||||
AND platform <> UPPER(platform);
|
||||
"""
|
||||
cur.execute(update_sql)
|
||||
affected_rows = cur.rowcount
|
||||
conn.commit()
|
||||
log_msg(f"platform 大写转换完成,共 {affected_rows} 行受影响", "success")
|
||||
return affected_rows
|
||||
except Exception as e:
|
||||
log_msg(f"platform 大写转换失败:{e}", "error")
|
||||
if conn:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
if cur:
|
||||
cur.close()
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
|
||||
@flow(name="platform-order-upper-sync", log_prints=True)
|
||||
def platform_order_upper_sync() -> int:
|
||||
return update_platform_upper()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
platform_order_upper_sync.serve(name="platform-order-upper-sync", cron="*/2 * * * *")
|
||||
@@ -22,7 +22,7 @@ RIIN_LOGIN_URL = "http://tshirt.riin.com/auth/api/auth/login"
|
||||
RIIN_ORDER_URL = "https://tshirt.riin.com/distribution/order/page"
|
||||
USERNAME_RIIN = "预上网"
|
||||
PASSWORD_RIIN = "Aa123456."
|
||||
SUB_ID_LIST = ["10368", "10283", "16905", "11502", "13615", "11531"]
|
||||
SUB_ID_LIST = ["10368", "10283", "16905", "11502", "13615", "11531","12279"]
|
||||
MAX_RETRY = 3; REQUEST_TIMEOUT = 30; PAGE_SIZE_RIIN = 200
|
||||
|
||||
LOOKBACK_DAYS = 6 # 回查14天避免漏单
|
||||
@@ -208,4 +208,5 @@ async def riin_platform_sync():
|
||||
log_msg("===== 完成 =====")
|
||||
|
||||
if __name__ == "__main__":
|
||||
riin_platform_sync.serve(name="platform-order-riin", cron="15 * * * *")
|
||||
# 直跑一次用于本地调试;勿改回 serve():同名 upsert 会覆盖 server 上挂 work pool 的正式 deployment
|
||||
riin_platform_sync()
|
||||
|
||||
@@ -273,4 +273,5 @@ def rouhui_platform_sync():
|
||||
log_msg("===== 提取完成 =====")
|
||||
|
||||
if __name__ == "__main__":
|
||||
rouhui_platform_sync.serve(name="platform-order-rouhui", cron="*/15 * * * *")
|
||||
# 直跑一次用于本地调试;勿改回 serve():同名 upsert 会覆盖 server 上挂 work pool 的正式 deployment
|
||||
rouhui_platform_sync()
|
||||
@@ -359,10 +359,18 @@ def sd_format_item(item):
|
||||
create_time = item.get("gmtCreated", "")
|
||||
parcel_list = item.get("parcelList", [])
|
||||
if parcel_list and isinstance(parcel_list, list):
|
||||
qty = sum(p.get("totalQty", 0) for p in parcel_list)
|
||||
def safe_int(val):
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return 0
|
||||
|
||||
qty = sum(safe_int(p.get("totalQty", 0)) for p in parcel_list)
|
||||
else:
|
||||
qty = 0
|
||||
|
||||
|
||||
|
||||
return [
|
||||
to_chinese_country(country),
|
||||
to_chinese_country(country),
|
||||
@@ -471,4 +479,5 @@ async def sdspod_platform_sync():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sdspod_platform_sync.serve(name="platform-order-sdspod", cron="*/15 * * * *")
|
||||
# 直跑一次用于本地调试;勿改回 serve():同名 upsert 会覆盖 server 上挂 work pool 的正式 deployment
|
||||
sdspod_platform_sync()
|
||||
+49
-8
@@ -33,6 +33,8 @@ RIIN_EXPRESS_SOURCE_MAP: dict[int, str] = {
|
||||
RIIN_GOODS_STATUS_MAP: dict[str, str] = {
|
||||
"NOT_SHIPPED": "待发货", "SHIPPED": "已发货", "CANCEL": "已取消",
|
||||
"CANCELLED": "已取消", "REFUNDING": "退款中",
|
||||
# 商品级状态复用平台状态枚举,COMPLETE/PAUSE 缺失曾导致 1/3 商品行 status 为 NULL
|
||||
"COMPLETE": "已完成", "PAUSE": "暂停",
|
||||
}
|
||||
|
||||
RIIN_PLATFORM_STATUS_MAP: dict[str, str] = {
|
||||
@@ -55,6 +57,17 @@ SDS_CHANNEL_TYPE_LABEL_MAP: dict[str, str] = {
|
||||
|
||||
# ---------- 日期解析 ----------
|
||||
|
||||
# 未知枚举值告警:同值只报一次,避免批量同步刷屏(对齐 TS 版 warnUnknown)
|
||||
_warned_values: set[str] = set()
|
||||
|
||||
|
||||
def _warn_unknown(map_name: str, value: object) -> None:
|
||||
key = f"{map_name}:{value!r}"
|
||||
if key not in _warned_values:
|
||||
_warned_values.add(key)
|
||||
print(f"[mapper] unknown {map_name} value: {value!r}")
|
||||
|
||||
|
||||
def safe_date(value: Any, context: str = "") -> datetime:
|
||||
"""解析日期字符串(无时区视为 +08:00),返回 tz-aware datetime。"""
|
||||
if not value or not isinstance(value, str):
|
||||
@@ -188,15 +201,20 @@ class GoodsUpsertInput:
|
||||
|
||||
# ---------- RIIN 映射 ----------
|
||||
|
||||
def map_riin_system_status(order_status: int) -> str | None:
|
||||
return RIIN_SYSTEM_STATUS_MAP.get(order_status)
|
||||
def map_riin_system_status(order_status: int | None) -> str | None:
|
||||
if order_status is None:
|
||||
return None
|
||||
v = RIIN_SYSTEM_STATUS_MAP.get(order_status)
|
||||
if v is None:
|
||||
_warn_unknown("RIIN systemStatus", order_status)
|
||||
return v
|
||||
|
||||
|
||||
def map_riin_order(raw: Any) -> OrderUpsertInput:
|
||||
platform_oid = raw.platform_oid
|
||||
ordered_at = _parse(raw.push_time, f"RIIN order {platform_oid} pushTime")
|
||||
confirmed_at = _parse(raw.distributor_push_time, f"RIIN order {platform_oid} distributorPushTime")
|
||||
system_status = map_riin_system_status(raw.order_status) if raw.order_status is not None else None
|
||||
system_status = map_riin_system_status(raw.order_status)
|
||||
is_terminal = system_status in ("已发货", "已关闭", "已取消")
|
||||
completed_at: datetime | None = None
|
||||
if is_terminal:
|
||||
@@ -236,6 +254,15 @@ def map_riin_order(raw: Any) -> OrderUpsertInput:
|
||||
)
|
||||
|
||||
|
||||
def map_riin_goods_status(goods_status: str | None) -> str | None:
|
||||
if not goods_status:
|
||||
return None
|
||||
v = RIIN_GOODS_STATUS_MAP.get(goods_status)
|
||||
if v is None:
|
||||
_warn_unknown("RIIN goodsStatus", goods_status)
|
||||
return v
|
||||
|
||||
|
||||
def map_riin_goods(raw: Any, parent_order_no: str, parent_created_at: datetime) -> GoodsUpsertInput:
|
||||
product_code = raw.product_code
|
||||
item_name = product_code.split("-")[0] if product_code else None
|
||||
@@ -250,7 +277,7 @@ def map_riin_goods(raw: Any, parent_order_no: str, parent_created_at: datetime)
|
||||
itemSize=raw.size_code,
|
||||
itemStyleNo=style_no,
|
||||
quantity=raw.num if raw.num is not None else 1,
|
||||
status=RIIN_GOODS_STATUS_MAP.get(raw.goods_status) if raw.goods_status else None,
|
||||
status=map_riin_goods_status(raw.goods_status),
|
||||
)
|
||||
|
||||
|
||||
@@ -262,13 +289,27 @@ def map_sds_channel_type(channel_type: Any) -> str | None:
|
||||
return SDS_CHANNEL_TYPE_LABEL_MAP.get(channel_type)
|
||||
|
||||
|
||||
def map_sds_system_status(status: int) -> str | None:
|
||||
return SDS_SYSTEM_STATUS_MAP.get(status)
|
||||
def map_sds_system_status(status: int | None) -> str | None:
|
||||
if status is None:
|
||||
return None
|
||||
v = SDS_SYSTEM_STATUS_MAP.get(status)
|
||||
if v is None:
|
||||
_warn_unknown("SDS systemStatus", status)
|
||||
return v
|
||||
|
||||
|
||||
def map_sds_goods_status(status: int | None) -> str | None:
|
||||
if status is None:
|
||||
return None
|
||||
v = SDS_GOODS_STATUS_MAP.get(status)
|
||||
if v is None:
|
||||
_warn_unknown("SDS goodsStatus", status)
|
||||
return v
|
||||
|
||||
|
||||
def map_sds_order(raw: Any, channel_type_map: dict[int, str | None], pre_courier: str | None = None) -> OrderUpsertInput:
|
||||
ordered_at = _parse(raw.gmt_pay, f"SDS order {raw.no} gmtPay")
|
||||
system_status = map_sds_system_status(raw.status) if raw.status is not None else None
|
||||
system_status = map_sds_system_status(raw.status)
|
||||
is_terminal = system_status in ("已完成", "已取消")
|
||||
completed_at: datetime | None = None
|
||||
if is_terminal and raw.finish_time and raw.finish_time > 0:
|
||||
@@ -369,7 +410,7 @@ def map_sds_goods(raw: Any, parent_order_no: str, parent_created_at: datetime) -
|
||||
itemSize=raw.product_size,
|
||||
itemStyleNo=_parse_sds_style_no(product_name),
|
||||
quantity=raw.num if raw.num is not None else 1,
|
||||
status=SDS_GOODS_STATUS_MAP.get(raw.status) if raw.status is not None else None,
|
||||
status=map_sds_goods_status(raw.status),
|
||||
)
|
||||
|
||||
|
||||
|
||||
+8
-1
@@ -97,7 +97,14 @@ deployments:
|
||||
- name: platform-order-sdspod
|
||||
entrypoint: flows/platform_order/sdspod.py:sdspod_platform_sync
|
||||
schedules:
|
||||
- cron: '*/30 * * * *'
|
||||
- cron: 0 * * * *
|
||||
timezone: Asia/Shanghai
|
||||
work_pool:
|
||||
name: inkreach-pool
|
||||
- name: platform-order-upper-sync
|
||||
entrypoint: flows/platform_order/platform_order_upper_sync.py:platform_order_upper_sync
|
||||
schedules:
|
||||
- cron: '*/2 * * * *'
|
||||
timezone: Asia/Shanghai
|
||||
work_pool:
|
||||
name: inkreach-pool
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""一次性回填:RIIN Goods.status 为 NULL 的历史行(终态"已发货"订单)。
|
||||
|
||||
背景:RIIN 商品级 goods_status 复用平台状态枚举,COMPLETE/PAUSE 此前无映射,
|
||||
约 1/3 商品行 status 被写为 NULL(mappers.py 已补映射)。本脚本按订单终态
|
||||
将近似值回填;"已关闭"与非终态订单不在此列,走 riin-by-orders 重拉取真值。
|
||||
幂等:只更新 status IS NULL 的行。分批提交,避免长事务。
|
||||
|
||||
运行:uv run python scripts/backfill_riin_goods_status.py [--dry-run] [--batch N]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import psycopg2
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv(os.path.join(os.path.dirname(__file__), "..", ".env"))
|
||||
|
||||
DEFAULT_BATCH = 50_000
|
||||
|
||||
SQL_COUNT = """
|
||||
SELECT count(*) FROM "Goods" g
|
||||
JOIN "Orders" o ON o."orderNo" = g."orderNo" AND o."createdAt" = g."createdAt"
|
||||
WHERE o."systemName" = 'RIIN' AND o."systemStatus" = '已发货' AND g."status" IS NULL
|
||||
"""
|
||||
|
||||
# ctid 分批:PostgreSQL 的 UPDATE 不支持 LIMIT,用子查询圈定每批行
|
||||
SQL_BACKFILL_BATCH = """
|
||||
UPDATE "Goods" g SET "status" = '已发货'
|
||||
WHERE g.ctid IN (
|
||||
SELECT g2.ctid FROM "Goods" g2
|
||||
JOIN "Orders" o ON o."orderNo" = g2."orderNo" AND o."createdAt" = g2."createdAt"
|
||||
WHERE o."systemName" = 'RIIN' AND o."systemStatus" = '已发货' AND g2."status" IS NULL
|
||||
LIMIT %s
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="回填 RIIN Goods.status NULL(已发货订单)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="只统计待回填行数,不写入")
|
||||
parser.add_argument("--batch", type=int, default=DEFAULT_BATCH, help="每事务回填行数")
|
||||
args = parser.parse_args()
|
||||
|
||||
conn = psycopg2.connect(os.environ["DB_DSN"])
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(SQL_COUNT)
|
||||
(pending,) = cur.fetchone()
|
||||
print(f"[backfill] 待回填(status IS NULL 且订单已发货): {pending} 行")
|
||||
if args.dry_run or pending == 0:
|
||||
return
|
||||
|
||||
total = 0
|
||||
while True:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(SQL_BACKFILL_BATCH, (args.batch,))
|
||||
updated = cur.rowcount
|
||||
conn.commit()
|
||||
total += updated
|
||||
print(f"[backfill] 本批 {updated} 行,累计 {total}")
|
||||
if updated < args.batch:
|
||||
break
|
||||
print(f"[backfill] 完成,共回填 {total} 行")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,125 @@
|
||||
"""mappers 状态映射单测。
|
||||
|
||||
运行:uv run python -m unittest discover -s tests -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import unittest
|
||||
from contextlib import redirect_stdout
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
from flows.shared import mappers
|
||||
from flows.shared.mappers import (
|
||||
map_riin_goods,
|
||||
map_riin_goods_status,
|
||||
map_riin_system_status,
|
||||
map_sds_goods_status,
|
||||
map_sds_system_status,
|
||||
)
|
||||
|
||||
TZ_CN = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
class WarnUnknownTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
mappers._warned_values.clear()
|
||||
|
||||
def _capture(self):
|
||||
return io.StringIO()
|
||||
|
||||
def test_未知值告警且同值只报一次(self):
|
||||
buf = self._capture()
|
||||
with redirect_stdout(buf):
|
||||
self.assertIsNone(map_riin_goods_status("SOMETHING_NEW"))
|
||||
self.assertIsNone(map_riin_goods_status("SOMETHING_NEW")) # 第二次不再报
|
||||
lines = [ln for ln in buf.getvalue().splitlines() if "SOMETHING_NEW" in ln]
|
||||
self.assertEqual(len(lines), 1)
|
||||
self.assertIn("[mapper] unknown RIIN goodsStatus", lines[0])
|
||||
|
||||
|
||||
class RiinGoodsStatusTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
mappers._warned_values.clear()
|
||||
|
||||
def test_全部已知枚举可映射(self):
|
||||
cases = {
|
||||
"NOT_SHIPPED": "待发货",
|
||||
"SHIPPED": "已发货",
|
||||
"CANCEL": "已取消",
|
||||
"CANCELLED": "已取消",
|
||||
"REFUNDING": "退款中",
|
||||
"COMPLETE": "已完成", # 本次补齐(曾导致 NULL)
|
||||
"PAUSE": "暂停", # 本次补齐(曾导致 NULL)
|
||||
}
|
||||
for raw, want in cases.items():
|
||||
with self.subTest(raw=raw):
|
||||
self.assertEqual(map_riin_goods_status(raw), want)
|
||||
|
||||
def test_空值返回None(self):
|
||||
self.assertIsNone(map_riin_goods_status(None))
|
||||
self.assertIsNone(map_riin_goods_status(""))
|
||||
|
||||
def test_map_riin_goods_接线_PAUSE与COMPLETE不再落NULL(self):
|
||||
# 复现线上订单 GSU1S555400MFLL:4 行商品 goods_status=PAUSE/COMPLETE
|
||||
created = datetime(2026, 9, 14, 9, 39, 24, tzinfo=TZ_CN)
|
||||
base = dict(
|
||||
platform_oid="GSU1S555400MFLL", product_code="DG001-BK",
|
||||
color_code="BK", size_code="L", style_code=None, num=1,
|
||||
)
|
||||
g1 = map_riin_goods(SimpleNamespace(id="P1", goods_status="PAUSE", **base),
|
||||
"GSU1S555400MFLL", created)
|
||||
g2 = map_riin_goods(SimpleNamespace(id="P2", goods_status="COMPLETE", **base),
|
||||
"GSU1S555400MFLL", created)
|
||||
self.assertEqual(g1.status, "暂停")
|
||||
self.assertEqual(g2.status, "已完成")
|
||||
self.assertEqual(g1.productionNo, "P1")
|
||||
self.assertEqual(g1.quantity, 1)
|
||||
self.assertEqual(g1.itemName, "DG001")
|
||||
|
||||
|
||||
class RiinSystemStatusTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
mappers._warned_values.clear()
|
||||
|
||||
def test_已知与空值(self):
|
||||
self.assertEqual(map_riin_system_status(12), "已发货")
|
||||
self.assertEqual(map_riin_system_status(-4), "待推送")
|
||||
self.assertIsNone(map_riin_system_status(None))
|
||||
|
||||
def test_未知值返回None并告警(self):
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
self.assertIsNone(map_riin_system_status(999))
|
||||
self.assertIn("unknown RIIN systemStatus", buf.getvalue())
|
||||
|
||||
|
||||
class SdsStatusTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
mappers._warned_values.clear()
|
||||
|
||||
def test_商品状态含已删除(self):
|
||||
self.assertEqual(map_sds_goods_status(99), "已删除")
|
||||
self.assertEqual(map_sds_goods_status(98), "已取消")
|
||||
self.assertEqual(map_sds_goods_status(4), "已完成")
|
||||
self.assertIsNone(map_sds_goods_status(None))
|
||||
|
||||
def test_未知商品状态返回None并告警(self):
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
self.assertIsNone(map_sds_goods_status(6))
|
||||
self.assertIn("unknown SDS goodsStatus", buf.getvalue())
|
||||
|
||||
def test_订单状态与未知告警(self):
|
||||
self.assertEqual(map_sds_system_status(3), "备货中")
|
||||
self.assertIsNone(map_sds_system_status(None))
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
self.assertIsNone(map_sds_system_status(42))
|
||||
self.assertIn("unknown SDS systemStatus", buf.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user