inkreachhub/inkreach-prefect
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 通知发送层切换到群机器人 webhook, 排版精简

Browse Source 
- wecom.py 重写: HTTP POST webhook(替代 aibot WebSocket 按需连接), 接口 send_markdown/send_text 不变, Secret Block wecom-webhook 存 URL
- webhook markdown 支持标题/引用/font 三色, 报告改为 '## 标题 + 单行结论 + 引用详情' 紧凑排版, 结论行用 info/warning 色
- alert_failure 改标题式排版; device_check 结论用 font 色, 去掉负载行
- 移除 wecom-aibot-sdk 依赖与 wecom_capture_chatid.py(镜像下次构建生效, 无需立即重建)
This commit is contained in:
yeuimu
2026-09-07 10:09:11 +00:00
parent c545c1b87d
commit 9b130097f9
8 changed files with 283 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flows/monitor/device_check.py
+42 -51
View File
@@ -291,11 +291,11 @@ def _check_services() -> dict:
def _mark(pct: float | None, threshold: int) -> str:
"""百分比展示:超阈值加粗+🔴(WeCom aibot markdown 不支持 font 标签和表格)。"""
"""百分比展示:超阈值用 warning 色加粗+🔴。"""
if pct is None:
return "—"
if pct >= threshold:
return f"**{pct:.0f}%**🔴"
return f'<font color="warning">**{pct:.0f}%** 🔴</font>'
return f"{pct:.0f}%"
@@ -343,104 +343,94 @@ def device_check(alarm_threshold: int = 80) -> dict:
print(f"[device-check] 磁盘检查失败: {err}")
threshold = alarm_threshold
header = f"🖥 **设备日报** {datetime.now():%m-%d %H:%M}（过去 24h · 采样 {agg['count']} 次"
header = f"## 🖥 设备日报 {datetime.now():%m-%d %H:%M}（24h 采样 {agg['count']} 次"
if agg.get("firstTs"):
first = datetime.fromisoformat(agg["firstTs"])
if first > datetime.now() - timedelta(hours=23):
header += f" · 自 {first:%m-%d %H:%M} 起"
lines = [header + "）", ""]
lines = [header + "）"]
# ---- 内存 / swap ----
mem, swap = agg.get("mem"), agg.get("swap")
if mem:
lines.append(f"**内存** 平均 {mem['avg']:.0f}% · 峰值 {_mark(mem['peak'], threshold)} · 最低 {mem['min']:.0f}%"
+ (f"（共 {_gb((agg.get('memTotal') or 0) * (1 << 30))}）" if agg.get("memTotal") else ""))
total_txt = f"（共 {_gb((agg.get('memTotal') or 0) * (1 << 30))}）" if agg.get("memTotal") else ""
lines.append(f"**内存** 均 {mem['avg']:.0f}% · 峰 {_mark(mem['peak'], threshold)} · 低 {mem['min']:.0f}%{total_txt}")
else:
try:
now_mem = _check_memory()
lines.append(f"**内存** 当前 {_mark(now_mem['pct'], threshold)}（{now_mem['used']}/{now_mem['total']}，无采样数据）")
lines.append(f"**内存** 当前 {_mark(now_mem['pct'], threshold)}（{now_mem['used']}/{now_mem['total']}，无采样）")
mem = {"peak": now_mem["pct"]}
except Exception: # noqa: BLE001
lines.append("**内存** 检查失败")
mem = None
if swap:
lines.append(f"**交换内存** 平均 {swap['avg']:.0f}% · 峰值 {_mark(swap['peak'], threshold)} · 最低 {swap['min']:.0f}%"
+ (f"（共 {_gb((agg.get('swapTotal') or 0) * (1 << 30))}）" if agg.get("swapTotal") else ""))
total_txt = f"（共 {_gb((agg.get('swapTotal') or 0) * (1 << 30))}）" if agg.get("swapTotal") else ""
lines.append(f"**交换内存** 均 {swap['avg']:.0f}% · 峰 {_mark(swap['peak'], threshold)} · 低 {swap['min']:.0f}%{total_txt}")
else:
lines.append("**交换内存** 无采样数据")
swap = None
# ---- 磁盘:总体 / 根目录 / home ----
lines.append("**磁盘**")
if overall:
lines.append(f"> 总体 已用 {_gb(overall['used'])} · 剩余 {_gb(overall['free'])}（{_mark(overall['pct'], threshold)}）")
lines.append(f"**磁盘** 总体 已用 {_gb(overall['used'])} · 剩 {_gb(overall['free'])}"
f"（{_mark(overall['pct'], threshold)}）")
else:
lines.append("**磁盘** 检查不可用")
label_map = {"/": "root", "/home": "home"}
for m in mounts:
g = growth.get(label_map.get(m["label"], ""))
growth_txt = ""
if g and g.get("dir"):
growth_txt = f"｜增长最快 {g['dir']} {_fmt_growth(g['growth'])}"
elif m["label"] in ("/", "/home"):
growth_txt = "｜增长数据不足" if not g else ""
lines.append(f"> {m['label']} 已用 {_gb(m['used'])} · 剩余 {_gb(m['free'])}"
f"（{_mark(m['pct'], threshold)}）{growth_txt}")
if not mounts:
lines.append("> 磁盘检查不可用")
growth_txt = f" · 增长最快 {g['dir']} {_fmt_growth(g['growth'])}" if g and g.get("dir") else ""
lines.append(f"> {m['label']} {_mark(m['pct'], threshold)} · 已用 {_gb(m['used'])} · 剩 {_gb(m['free'])}{growth_txt}")
# ---- GPU ----
gpu_stats = agg.get("gpus") or []
if gpu_stats:
for g in gpu_stats:
name = f"{g['name']}（GPU{g['index']}）" if g.get("name") else f"GPU{g['index']}"
name = g.get("name") or f"GPU{g['index']}"
util = g.get("util")
util_txt = f"利用率 平均 {util['avg']:.0f}% · 峰值 {util['peak']:.0f}% · 最低 {util['min']:.0f}%" if util else "利用率 无数据"
mem_peak = g.get("memPeakUsed")
mem_total = g.get("memTotal")
util_txt = (f"利用率 均 {util['avg']:.0f}% · 峰 {util['peak']:.0f}% · 低 {util['min']:.0f}%"
if util else "利用率 无数据")
mem_peak, mem_total = g.get("memPeakUsed"), g.get("memTotal")
if mem_peak is not None and mem_total:
mem_pct = mem_peak / mem_total * 100
mem_txt = f"显存峰值 {_gb(mem_peak * (1 << 30))}/{_gb(mem_total * (1 << 30))}（{_mark(mem_pct, threshold)}）"
mem_txt = (f"显存峰 {_gb(mem_peak * (1 << 30))}/{_gb(mem_total * (1 << 30))}"
f"（{_mark(mem_peak / mem_total * 100, threshold)}）")
else:
mem_txt = "显存 无数据"
temp_peak = g.get("tempPeak")
if temp_peak is not None:
temp_txt = f"温度峰值 {temp_peak:.0f}°C" + ("🔴 过热" if temp_peak >= GPU_TEMP_WARN else " 正常 ✅")
else:
temp_txt = "温度 无数据"
lines.append(f"**GPU** {name}｜{util_txt}｜{mem_txt}｜{temp_txt}")
temp_txt = (f"温度峰 {temp_peak:.0f}°C" + (" 🔴过热" if temp_peak >= GPU_TEMP_WARN else " ✅")
if temp_peak is not None else "温度 无数据")
lines.append(f"**GPU** {name} {util_txt} ｜ {mem_txt} ｜ {temp_txt}")
else:
gpus_now = _check_gpus()
if isinstance(gpus_now, list) and gpus_now:
for g in gpus_now:
lines.append(f"**GPU** {g['name']}｜利用率 {g['utilPct']:.0f}%｜显存 {g['memUsed']}/{g['memTotal']}"
f"（无采样数据）")
lines.append(f"**GPU** {g['name']} 利用率 {g['utilPct']:.0f}% · 显存 {g['memUsed']}/{g['memTotal']}（无采样）")
else:
lines.append(f"**GPU** {gpus_now if isinstance(gpus_now, str) else '未检测到'}")
# ---- Docker / 失败任务 / 探活 ----
# ---- Docker / 探活 / 失败任务 ----
docker = checks.get("docker") or {}
issues = docker.get("issues") or []
svc = checks.get("services") or {}
if docker.get("running") is None:
lines.append(f"**Docker** {issues[0]['issue'] if issues else '不可用'}")
docker_txt = issues[0]["issue"] if issues else "不可用"
else:
docker_line = f"**Docker** 运行 {docker['running']}/{docker.get('total', '?')}"
if docker.get("stopped"):
docker_line += f" · 已停用 {docker['stopped']}"
docker_line += f" · 异常 **{len(issues)}**🔴" if issues else " · 异常 0"
lines.append(docker_line)
docker_txt = f"运行 {docker['running']}/{docker.get('total', '?')} · 异常 {len(issues)}" + ("🔴" if issues else "")
if svc:
docker_txt += f" ｜ OCR {'✅' if svc.get('ocr') else '🔴'} ｜ PG {'✅' if svc.get('pg') else '🔴'}"
lines.append(f"**Docker** {docker_txt}")
for it in issues[:8]:
lines.append(f"> 🔴 {it['name']}：{it['issue']}")
failures = checks.get("prefectFailures") or []
lines.append(f"**失败任务(24h)** {len(failures)} 次" + (" ✅" if not failures else "🔴"))
for f in failures[:5]:
lines.append(f"> 🔴 {f['flow']} ·「{f['name']}」{f['state']} · {f['start']}")
if len(failures) > 5:
lines.append(f"> …共 {len(failures)} 次")
svc = checks.get("services") or {}
if svc:
lines.append(f"**探活** OCR {'✅' if svc.get('ocr') else '🔴'}｜PG {'✅' if svc.get('pg') else '🔴'}"
f"｜负载 {os.getloadavg()[0]:.1f}")
if failures:
lines.append(f"**失败任务(24h)** {len(failures)} 次 🔴")
for f in failures[:5]:
lines.append(f"> {f['flow']}「{f['name']}」{f['state']} · {(f.get('start') or '')[5:]}")
if len(failures) > 5:
lines.append(f"> …共 {len(failures)} 次")
else:
lines.append("**失败任务(24h)** 0 ✅")
mem_peak_for_alarm = mem.get("peak") if mem else None
swap_peak_for_alarm = swap.get("peak") if swap else None
@@ -452,7 +442,8 @@ def device_check(alarm_threshold: int = 80) -> dict:
or any((g.get("tempPeak") or 0) >= GPU_TEMP_WARN for g in gpu_stats)
or bool(issues) or bool(failures) or not svc.get("ocr") or not svc.get("pg")
)
lines += ["", ("🔴 **有项目需要处理，见上**" if alarms else "✅ **一切正常**")]
lines.append('<font color="warning">🔴 有项目需要处理，见上</font>' if alarms
else '<font color="info">✅ 一切正常</font>')
content = "\n".join(lines)
sent = send_markdown(content)