feat: 四份报告按用户口径重做
- 核验日报: 先报订单数量(远程vs本地, verify 修复后重数 verifyReport.post)/商品行数一致性, 再报状态不符单数; SDS 统计口径排除'售后'与本地对称 - OCR: Orders 加 ocrProcessedAt 列(写入自动盖时间戳), 日报报'今日处理 X · 识别失败 Y'(db.count_ocr_today) - 失败任务: check_prefect_failures 补 workflow 名, 日报/设备报列 run 名+状态+时间 - 设备日报: 新增 metrics-agent 侧车(每5分钟采样内存/swap/GPU+每日 du 快照), 报告改 平均/峰值/最低 + 总体/根目录/home 余量 + 增长最快目录 + GPU 温度; 采样缺失降级瞬时值 - Dockerfile 加 tzdata(TZ=Asia/Shanghai); du 用实际占块且容错 rc=1
This commit is contained in:
yeuimu
@@ -1,12 +1,12 @@
|
||||
"""数据核验日报(每日 18:30):近 N 天 RIIN/SDS 订单+商品数据准确性核验并上报企微。
|
||||
"""数据核验日报(每日 18:30):RIIN/SDS 数量一致性 + 状态差异 + OCR 当日 + 失败任务回顾。
|
||||
|
||||
对应 daily-report deployment,cron 30 18 * * *。内容:
|
||||
1. 复用 riin-verify / sds-verify 逻辑(subflow 调用):远程 vs 本地逐天逐状态计数比对,
|
||||
有差异自动拉取修复——与早 6/7 点的每日核验互补,二次确认白天同步质量。
|
||||
2. 商品数据完整性:窗口内零商品订单 / 商品无名称 / 全库孤儿商品(本地 SQL)。
|
||||
3. OCR 处理状态与当日失败 flow runs 回顾(事件报警的兜底汇总)。
|
||||
对应 daily-report deployment,cron 30 18 * * *。报告结构:
|
||||
1. RIIN/SDS:先报订单数量(远程 vs 本地,有差异修复后重数)与商品行数是否一致;
|
||||
再报核验时发现多少单状态不符(已自动刷新修复)。subflow 复用 riin/sds-verify。
|
||||
2. OCR:今天处理了多少订单、识别失败多少(按 ocrProcessedAt 盖章统计)。
|
||||
3. 失败任务:近 24h 哪些 workflow、哪些 run、失败几次、何时失败(事件报警的兜底汇总)。
|
||||
|
||||
消息排版遵循 WeCom markdown 子集(无表格,引用块+彩色字体)。
|
||||
消息排版遵循 WeCom aibot markdown 子集(无表格、无 font 标签,引用块 + emoji)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -21,70 +21,79 @@ from flows.sds.verify import sds_verify
|
||||
from flows.shared import SyncDB, emit_sync_summary, send_markdown, start_of_day
|
||||
|
||||
|
||||
|
||||
def _short_date(date_str: str) -> str:
|
||||
return date_str[5:] if len(date_str) == 10 and date_str[:4].isdigit() else date_str
|
||||
|
||||
|
||||
def _affected_units(diff: dict) -> int:
|
||||
"""状态不符单数 ≈ 本地比远程多的状态计数之和(远程改了状态的单)。"""
|
||||
return sum(v for v in (diff or {}).values() if v > 0)
|
||||
|
||||
|
||||
def _system_lines(name: str, report: dict, goods: dict, lines: list[str]) -> bool:
|
||||
"""单系统的核验两段:①数量是否一致(订单+商品) ②状态不符多少单。返回是否需关注。"""
|
||||
vr = (report.get("verifyReport") or {}) if isinstance(report, dict) else {}
|
||||
days_r = vr.get("days") or []
|
||||
post = vr.get("post")
|
||||
if post:
|
||||
remote_t, local_t, ok = post["remote"], post["local"], post.get("daysWithDiff", 0) == 0
|
||||
else:
|
||||
remote_t = sum(d.get("remote", 0) for d in days_r)
|
||||
local_t = sum(d.get("local", 0) for d in days_r)
|
||||
ok = all(d.get("ok") for d in days_r)
|
||||
|
||||
if ok:
|
||||
lines.append(f"**{name} 订单** 远程 {remote_t:,} · 本地 {local_t:,} ✅")
|
||||
else:
|
||||
lines.append(f"**{name} 订单** 远程 {remote_t:,} · 本地 {local_t:,} ⚠️ 不一致")
|
||||
|
||||
diff_days = [d for d in days_r if not d.get("ok")]
|
||||
diff_units = sum(_affected_units(d.get("diff")) for d in diff_days)
|
||||
if diff_units:
|
||||
lines.append(f"> 核验时发现约 {diff_units:,} 单状态不符（{len(diff_days)} 天），已自动刷新修复")
|
||||
|
||||
g = goods or {}
|
||||
zero = g.get("zeroGoodsOrders", 0)
|
||||
if zero:
|
||||
lines.append(f"**{name} 商品** {g.get('goods', 0):,} 行 · {zero:,} 单缺商品 ⚠️")
|
||||
return True
|
||||
lines.append(f"**{name} 商品** {g.get('goods', 0):,} 行 · 无缺漏 ✅")
|
||||
return not ok
|
||||
|
||||
|
||||
def build_daily_message(
|
||||
days: int, window: str, riin: dict, sds: dict,
|
||||
goods: dict, ocr: dict, failures: list[dict],
|
||||
goods: dict, ocr_today: dict, ocr_pending: int, failures: list[dict],
|
||||
) -> tuple[str, bool]:
|
||||
"""拼装日报 markdown,返回 (消息, 是否需关注)。结论式极简排版,不暴露内部术语。"""
|
||||
"""拼装日报 markdown,返回 (消息, 是否需关注)。"""
|
||||
lines = [f"📊 **数据核验日报** {datetime.now().strftime('%m-%d %H:%M')}"
|
||||
f"（近 {days} 天，{window}）", ""]
|
||||
attention = False
|
||||
diff_names: list[str] = []
|
||||
|
||||
for name, report in (("RIIN", riin), ("SDS", sds)):
|
||||
try:
|
||||
days_r = (report.get("verifyReport") or {}).get("days") or []
|
||||
summary = (report.get("verifyReport") or {}).get("summary") or {}
|
||||
diff_days = [d for d in days_r if not d.get("ok")]
|
||||
total = summary.get("totalDays", len(days_r))
|
||||
if not diff_days:
|
||||
lines.append(f"**{name}** {total}/{total} 天与远程一致 ✅")
|
||||
else:
|
||||
diff_names.append(name)
|
||||
attention = True
|
||||
lines.append(f"**{name}** {total - len(diff_days)}/{total} 天与远程一致 ⚠️")
|
||||
if len(diff_days) <= 4:
|
||||
per_day = " · ".join(
|
||||
f"{_short_date(d['date'])} {_affected_units(d.get('diff'))}单" for d in diff_days)
|
||||
lines.append(f"> {per_day}状态不符，已自动重新同步")
|
||||
else:
|
||||
total_affected = sum(_affected_units(d.get("diff")) for d in diff_days)
|
||||
lines.append(f"> {len(diff_days)} 天共约 {total_affected:,} 单状态不符，已自动重新同步")
|
||||
attention |= _system_lines(name, report, (goods or {}).get(name), lines)
|
||||
except Exception as err: # noqa: BLE001
|
||||
lines.append(f"**{name}** 核验结果解析失败：{err}")
|
||||
attention = True
|
||||
lines.append("")
|
||||
|
||||
processed, failed = (ocr_today or {}).get("processed", 0), (ocr_today or {}).get("failed", 0)
|
||||
lines.append(f"**OCR 今日** 已处理 {processed:,} 单 · 识别失败 {failed:,} 单" + (" ✅" if not failed else ""))
|
||||
if ocr_pending:
|
||||
lines.append(f"> 累计待处理 {ocr_pending:,} 单")
|
||||
lines.append("")
|
||||
|
||||
zero_total = sum(g["zeroGoodsOrders"] for g in goods.values())
|
||||
lines += ["", f"**商品** RIIN {goods['RIIN']['orders']:,} 单/{goods['RIIN']['goods']:,} 行"
|
||||
f" · SDS {goods['SDS']['orders']:,} 单/{goods['SDS']['goods']:,} 行"
|
||||
+ (" ⚠️" if zero_total else " ✅")]
|
||||
if zero_total:
|
||||
lines.append(f"> 有 {zero_total} 单缺商品数据")
|
||||
attention = True
|
||||
lines.append(f"**OCR** 已完成 {ocr.get('done', 0):,} · 待处理 {ocr.get('pending', 0):,}"
|
||||
f" · 失败 {ocr.get('error', 0):,}")
|
||||
if failures:
|
||||
attention = True
|
||||
names = "、".join(f"{f['name']}（{f['state']}）" for f in failures[:4])
|
||||
lines.append(f"**失败任务(24h)** {len(failures)} 个：{names}")
|
||||
lines.append(f"**失败任务(24h)** 共 {len(failures)} 次 🔴")
|
||||
for f in failures[:6]:
|
||||
start = (f.get("start") or "")[5:] # MM-DD HH:MM
|
||||
lines.append(f"> {f.get('flow', '?')} · run「{f.get('name', '?')}」{f.get('state', '?')} · {start}")
|
||||
if len(failures) > 6:
|
||||
lines.append(f"> …等共 {len(failures)} 次")
|
||||
else:
|
||||
lines.append("**失败任务(24h)** 0 个 ✅")
|
||||
lines.append("**失败任务(24h)** 0 次 ✅")
|
||||
|
||||
lines.append("")
|
||||
if attention:
|
||||
lines.append("⚠️ **存在需关注项，见上**")
|
||||
if diff_names:
|
||||
lines.append("> 每天都出现的相同差异属远程统计口径问题，非数据丢失")
|
||||
else:
|
||||
lines.append("✅ **数据全部一致，无异常**")
|
||||
return "\n".join(lines), attention
|
||||
@@ -102,11 +111,13 @@ def daily_report(days: int = 7) -> dict:
|
||||
|
||||
db = SyncDB()
|
||||
goods = {sys: db.get_goods_integrity(sys, window_start, window_end) for sys in ("RIIN", "SDS")}
|
||||
ocr_today = db.count_ocr_today(today)
|
||||
ocr = db.count_ocr_status()
|
||||
failures = check_prefect_failures(hours=24)
|
||||
|
||||
window = f"{window_start:%m-%d}~{window_end:%m-%d}"
|
||||
content, attention = build_daily_message(days, window, riin, sds, goods, ocr, failures)
|
||||
content, attention = build_daily_message(days, window, riin, sds, goods,
|
||||
ocr_today, ocr.get("pending", 0), failures)
|
||||
|
||||
sent = send_markdown(content)
|
||||
emit_sync_summary("数据核验日报", [
|
||||
@@ -116,11 +127,12 @@ def daily_report(days: int = 7) -> dict:
|
||||
("SDS 零商品订单", goods["SDS"]["zeroGoodsOrders"]),
|
||||
("SDS 商品无名称", goods["SDS"]["goodsNoName"]),
|
||||
("全库孤儿商品", goods["RIIN"]["orphanGoods"]),
|
||||
("OCR 今日处理", ocr_today.get("processed", 0)), ("OCR 今日失败", ocr_today.get("failed", 0)),
|
||||
("OCR 待处理", ocr.get("pending", 0)), ("失败 runs(24h)", len(failures)),
|
||||
("已发企微", "是" if sent else "否"),
|
||||
])
|
||||
return {"attention": attention, "sent": sent,
|
||||
"goods": goods, "ocr": ocr, "failures24h": len(failures)}
|
||||
"goods": goods, "ocrToday": ocr_today, "failures24h": len(failures)}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user