feat: 接入企微智能机器人——异常报警/数据核验日报/设备健康日报
- flows/shared/wecom.py: WSClient 按需连接(连接→认证→发送→断开),全容错,超长分段 - flows/monitor/alert_failure.py: Prefect 事件触发(Failed/Crashed→run deployment),发群报警 - flows/monitor/daily_report.py: 每日 18:30 近7天 RIIN/SDS 核验(复用 verify)+商品完整性+OCR+失败回顾 - flows/monitor/device_check.py: 每日 09:00 内存/磁盘/GPU(80%阈值)+Docker 容器异常+服务探活 - db.py: 新增 get_goods_integrity/ping; compose worker 挂载 docker.sock+/:/host:ro+GPU - scripts/wecom_capture_chatid.py: 一次性捕获群 chat_id 工具
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""监控上报层:企微机器人异常报警 / 数据核验日报 / 设备健康日报。"""
|
||||
@@ -0,0 +1,129 @@
|
||||
"""企微机器人异常报警:任意 flow run 失败/崩溃时由 Prefect 事件触发器自动调用。
|
||||
|
||||
对应 wecom-alert deployment(prefect.yaml 里的 triggers 声明):
|
||||
- match: prefect.resource.id = prefect.flow-run.*
|
||||
- expect: prefect.flow-run.Failed / Crashed
|
||||
- parameters.event = "{{ event }}"
|
||||
|
||||
flow 体全容错——本 flow 自身永不进入 Failed 状态,避免自我触发循环。
|
||||
event 可能以 dict 或 str(jinja 渲染结果)传入,均做防御性解析;
|
||||
解析出 flow run id 后再查 Prefect API 补全权威信息(名称/错误消息/时间)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from prefect import flow
|
||||
|
||||
from flows.shared import send_markdown
|
||||
|
||||
UI_BASE = os.environ.get("PREFECT_UI_URL", "http://192.168.124.92:4200")
|
||||
|
||||
|
||||
def _parse_event(event: Any) -> dict:
|
||||
if isinstance(event, dict):
|
||||
return event
|
||||
if isinstance(event, str) and event.strip():
|
||||
# jinja "{{ event }}" 可能渲染成 JSON 串或 Python repr,统一抠除关键字段
|
||||
rid = re.search(r"prefect\.flow-run\.[0-9a-fA-F-]{36}", event)
|
||||
ev = re.search(r"prefect\.flow-run\.(?:Failed|Crashed)", event)
|
||||
name = re.search(r"prefect\.resource\.name['\"]?\s*[:=]\s*['\"]([^'\"]+)", event)
|
||||
occurred = re.search(r"occurred['\"]?\s*[:=]\s*['\"]([^'\"]+)", event)
|
||||
return {
|
||||
"event": ev.group(0) if ev else "",
|
||||
"resource": {
|
||||
"prefect.resource.id": rid.group(0) if rid else "",
|
||||
"prefect.resource.name": name.group(1) if name else "",
|
||||
},
|
||||
"occurred": occurred.group(1) if occurred else "",
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
def _flow_run_id(event: dict) -> str | None:
|
||||
rid = (event.get("resource") or {}).get("prefect.resource.id", "")
|
||||
m = re.match(r"^prefect\.flow-run\.(.+)$", rid)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _fetch_run_detail(run_id: str) -> dict:
|
||||
"""查 Prefect API 补全 flow run 权威信息,失败返回空 dict。"""
|
||||
api_url = os.environ.get("PREFECT_API_URL", "http://localhost:4200/api")
|
||||
try:
|
||||
resp = httpx.get(f"{api_url}/flow_runs/{run_id}", timeout=10)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[wecom-alert] 查询 flow run {run_id} 失败: {err}")
|
||||
return {}
|
||||
|
||||
|
||||
def _fmt_ts(value: Any) -> str:
|
||||
if not value:
|
||||
return "—"
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone().strftime("%m-%d %H:%M")
|
||||
except (ValueError, TypeError):
|
||||
return str(value)[:19]
|
||||
|
||||
|
||||
@flow(name="wecom-alert", log_prints=True)
|
||||
def wecom_alert_failure(event: dict | str | None = None) -> dict:
|
||||
"""Prefect 事件触发:flow run 失败/崩溃 → 发企微群报警。永不抛异常。"""
|
||||
result = {"sent": False, "flowName": None, "runId": None}
|
||||
try:
|
||||
ev = _parse_event(event)
|
||||
event_name = ev.get("event", "prefect.flow-run.?")
|
||||
resource_name = (ev.get("resource") or {}).get("prefect.resource.name", "")
|
||||
run_id = _flow_run_id(ev)
|
||||
|
||||
flow_name, run_name, state_msg, occurred = "", resource_name, "", ev.get("occurred", "")
|
||||
if run_id:
|
||||
detail = _fetch_run_detail(run_id)
|
||||
flow_name = detail.get("name", "") or resource_name
|
||||
run_name = detail.get("name", "")
|
||||
state = detail.get("state") or {}
|
||||
state_msg = (state.get("message") or "").strip()
|
||||
ended = detail.get("end_time") or detail.get("expected_start_time")
|
||||
occurred = occurred or ended
|
||||
|
||||
lines = [
|
||||
"🔴 **Prefect 流程失败报警**",
|
||||
f"- 事件: `{event_name}`",
|
||||
f"- Flow: {flow_name or resource_name or '未知'}",
|
||||
]
|
||||
if run_name and run_name != flow_name:
|
||||
lines.append(f"- Run: {run_name}")
|
||||
if state_msg:
|
||||
lines.append(f"- 错误: {state_msg[:200].replace(chr(10), ' ')}")
|
||||
lines.append(f"- 时间: {_fmt_ts(occurred)}")
|
||||
if run_id:
|
||||
lines.append(f"- 详情: [Prefect UI]({UI_BASE}/runs/flow-run/{run_id})")
|
||||
else:
|
||||
raw = json.dumps(ev, ensure_ascii=False)[:300] if ev else "(事件内容解析失败)"
|
||||
lines.append(f"- 原始事件: {raw}")
|
||||
|
||||
result["sent"] = send_markdown("\n".join(lines))
|
||||
result.update({"flowName": flow_name or None, "runId": run_id})
|
||||
print(f"[wecom-alert] 已发送失败报警: {flow_name or resource_name} ({event_name})")
|
||||
except Exception as err: # noqa: BLE001
|
||||
print(f"[wecom-alert] 报警流程异常(已吞掉,避免自我触发): {err}")
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 本地手测:模拟一条 Failed 事件
|
||||
wecom_alert_failure({
|
||||
"event": "prefect.flow-run.Failed",
|
||||
"resource": {
|
||||
"prefect.resource.id": "prefect.flow-run.00000000-0000-0000-0000-000000000000",
|
||||
"prefect.resource.name": "riin-order-sync/manual",
|
||||
},
|
||||
"occurred": datetime.now().astimezone().isoformat(),
|
||||
})
|
||||
@@ -0,0 +1,106 @@
|
||||
"""数据核验日报(每日 18:30):近 N 天 RIIN/SDS 订单+商品数据准确性核验并上报企微群。
|
||||
|
||||
对应 daily-report deployment,cron 30 18 * * *。内容:
|
||||
1. 复用 riin-verify / sds-verify 逻辑(subflow 调用):远程 vs 本地逐天逐状态计数比对,
|
||||
有差异自动拉取修复——与早 6/7 点的每日核验互补,二次确认白天同步质量。
|
||||
2. 商品数据完整性:窗口内零商品订单 / 商品无名称 / 全库孤儿商品(本地 SQL)。
|
||||
3. OCR 处理状态与当日失败 flow runs 回顾(事件报警的兜底汇总)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from prefect import flow
|
||||
|
||||
from flows.monitor.device_check import check_prefect_failures
|
||||
from flows.riin.verify import riin_verify
|
||||
from flows.sds.verify import sds_verify
|
||||
from flows.shared import SyncDB, emit_sync_summary, send_markdown, start_of_day
|
||||
|
||||
|
||||
def _verify_section(name: str, report: dict) -> tuple[str, list[str], bool]:
|
||||
"""把 verify flow 返回的 verifyReport 压缩成结论行 + 差异日明细行。"""
|
||||
days = (report.get("verifyReport") or {}).get("days") or []
|
||||
summary = (report.get("verifyReport") or {}).get("summary") or {}
|
||||
diff_days = [d for d in days if not d.get("ok")]
|
||||
total, n_diff = summary.get("totalDays", len(days)), summary.get("daysWithDiff", len(diff_days))
|
||||
head = f"**{name}**: {total} 天中 {total - n_diff} 天一致" + (" ✅" if not diff_days else f", {n_diff} 天有差异已修复")
|
||||
detail = [
|
||||
f"- {d['date']}: 本地 {d.get('local', '—')} / 远程 {d.get('remote', '—')},"
|
||||
f"差异 {d.get('diff') or '—'},已补拉 订单 {d.get('orders', 0)} / 商品 {d.get('goods', 0)}"
|
||||
for d in diff_days
|
||||
]
|
||||
totals = {k: v for k, v in summary.items() if k not in ("totalDays", "daysWithDiff")}
|
||||
if any(totals.values()):
|
||||
detail.append(f"- 修复量合计: {totals}")
|
||||
return head, detail, bool(diff_days)
|
||||
|
||||
|
||||
@flow(name="daily-report", log_prints=True)
|
||||
def daily_report(days: int = 7) -> dict:
|
||||
"""近 N 天数据核验 + 商品完整性 + OCR/失败回顾,发企微群日报。"""
|
||||
today = start_of_day(datetime.now())
|
||||
window_start = today - timedelta(days=days)
|
||||
window_end = today # 与 verify 的 recent 约定一致:近 N 天、不含今天
|
||||
|
||||
riin = riin_verify(mode="recent", recent_days=days)
|
||||
sds = sds_verify(mode="recent", recent_days=days)
|
||||
|
||||
db = SyncDB()
|
||||
goods = {sys: db.get_goods_integrity(sys, window_start, window_end) for sys in ("RIIN", "SDS")}
|
||||
ocr = db.count_ocr_status()
|
||||
|
||||
failures = check_prefect_failures(hours=24)
|
||||
|
||||
lines = [f"📊 **数据核验日报** {datetime.now().strftime('%Y-%m-%d %H:%M')}"
|
||||
f"(近 {days} 天,{window_start:%m-%d} ~ {window_end:%m-%d})", ""]
|
||||
|
||||
attention = False
|
||||
for name, report in (("RIIN", riin), ("SDS", sds)):
|
||||
try:
|
||||
head, detail, had_diff = _verify_section(name, report)
|
||||
lines.append(head)
|
||||
lines.extend(detail)
|
||||
attention = attention or had_diff
|
||||
except Exception as err: # noqa: BLE001
|
||||
lines.append(f"**{name}**: 核验结果解析失败: {err}")
|
||||
attention = True
|
||||
|
||||
lines.append("**商品完整性**:")
|
||||
lines.append("| 系统 | 订单 | 商品行 | 零商品订单 | 商品无名称 | 孤儿商品(全库) |")
|
||||
lines.append("| --- | --- | --- | --- | --- | --- |")
|
||||
for sys in ("RIIN", "SDS"):
|
||||
g = goods[sys]
|
||||
flag = " ⚠️" if g["zeroGoodsOrders"] > 0 else ""
|
||||
lines.append(f"| {sys} | {g['orders']} | {g['goods']} | {g['zeroGoodsOrders']}{flag} "
|
||||
f"| {g['goodsNoName']} | {g['orphanGoods']} |")
|
||||
if g["zeroGoodsOrders"] > 0:
|
||||
attention = True
|
||||
|
||||
lines.append(f"**OCR(SDS)**: 完成 {ocr.get('done', 0)} / 失败 {ocr.get('error', 0)} / "
|
||||
f"不支持 {ocr.get('unsupported', 0)} / 待处理 {ocr.get('pending', 0)}")
|
||||
lines.append(f"**近24h 失败 flow runs**: {len(failures)} 个" + ("" if failures else " ✅"))
|
||||
for f in failures[:8]:
|
||||
lines.append(f"- 🔴 {f['name']}({f['state']}) {f['start']} {f['message'][:60]}")
|
||||
if len(failures) > 8:
|
||||
lines.append(f"- ...共 {len(failures)} 个,详见 Prefect UI")
|
||||
if failures:
|
||||
attention = True
|
||||
|
||||
lines += ["", ("⚠️ **存在需关注项,见上**" if attention else "✅ **数据全部一致,无异常**")]
|
||||
|
||||
sent = send_markdown("\n".join(lines))
|
||||
emit_sync_summary("数据核验日报", [
|
||||
("核验窗口", f"近{days}天"), ("需关注", "是" if attention else "否"),
|
||||
("RIIN 零商品订单", goods["RIIN"]["zeroGoodsOrders"]),
|
||||
("SDS 零商品订单", goods["SDS"]["zeroGoodsOrders"]),
|
||||
("OCR 待处理", ocr.get("pending", 0)), ("失败 runs(24h)", len(failures)),
|
||||
("已发企微", "是" if sent else "否"),
|
||||
])
|
||||
return {"attention": attention, "sent": sent,
|
||||
"goods": goods, "ocr": ocr, "failures24h": len(failures)}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
daily_report.serve(name="daily-report", cron="30 18 * * *")
|
||||
@@ -0,0 +1,262 @@
|
||||
"""设备/系统健康日报(每日一次):内存/磁盘/GPU 使用率 + 系统级错误检查。
|
||||
|
||||
对应 device-check deployment,每天 09:00(可调)。检查项:
|
||||
- 内存/swap、宿主机磁盘(经 /:/host:ro 挂载)、GPU(nvidia-smi)——使用率 ≥ 阈值(默认 80%)标 🔴
|
||||
- Docker 容器状态(经 /var/run/docker.sock):应运行但退出 / restarting / unhealthy / OOMKilled
|
||||
- Prefect 近 24h FAILED/CRASHED flow runs、OCR 服务探活、PG 连通性、系统负载
|
||||
|
||||
依赖 docker-compose 给 prefect-worker 挂载 /:/host:ro、/var/run/docker.sock、GPU;
|
||||
未挂载时对应检查项自动降级为"不可用",不影响其余检查。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
from prefect import flow
|
||||
|
||||
from flows.shared import SyncDB, emit_sync_summary, send_markdown
|
||||
|
||||
# (容器内路径, 展示名);/host 为 compose 挂载的宿主机根目录
|
||||
DISK_PATHS = [("/host", "/"), ("/host/home", "/home")]
|
||||
DOCKER_SOCK = "/var/run/docker.sock"
|
||||
|
||||
_OCR_BASE = "http://192.168.124.92:9801"
|
||||
|
||||
|
||||
def _gb(nbytes: float) -> str:
|
||||
return f"{nbytes / (1 << 30):.1f}G"
|
||||
|
||||
|
||||
def _check_memory() -> dict:
|
||||
vm = psutil.virtual_memory()
|
||||
sw = psutil.swap_memory()
|
||||
return {
|
||||
"pct": vm.percent, "used": _gb(vm.used), "total": _gb(vm.total),
|
||||
"swapPct": sw.percent, "swapUsed": _gb(sw.used), "swapTotal": _gb(sw.total),
|
||||
}
|
||||
|
||||
|
||||
def _check_disks() -> list[dict]:
|
||||
paths = DISK_PATHS if os.path.isdir(DISK_PATHS[0][0]) else [("/", "/")]
|
||||
out = []
|
||||
for raw, label in paths:
|
||||
try:
|
||||
du = psutil.disk_usage(raw)
|
||||
out.append({"label": label, "pct": du.percent, "used": _gb(du.used), "total": _gb(du.total)})
|
||||
except Exception as err: # noqa: BLE001
|
||||
out.append({"label": label, "error": str(err)[:80]})
|
||||
return out
|
||||
|
||||
|
||||
def _check_gpus() -> list[dict] | str:
|
||||
"""nvidia-smi 查 GPU 利用率与显存。容器无 GPU 访问时返回说明文字。"""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["nvidia-smi", "--query-gpu=index,name,utilization.gpu,memory.used,memory.total",
|
||||
"--format=csv,noheader,nounits"],
|
||||
capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
return f"nvidia-smi 退出码 {proc.returncode}: {proc.stderr.strip()[:100]}"
|
||||
gpus = []
|
||||
for line in proc.stdout.strip().splitlines():
|
||||
parts = [p.strip() for p in line.split(",")]
|
||||
if len(parts) != 5:
|
||||
continue
|
||||
idx, name, util, mem_used, mem_total = parts
|
||||
mem_pct = float(mem_used) / float(mem_total) * 100 if float(mem_total) > 0 else 0.0
|
||||
gpus.append({"name": f"GPU{idx} {name[:24]}", "utilPct": float(util), "memPct": mem_pct,
|
||||
"memUsed": f"{float(mem_used)/1024:.1f}G", "memTotal": f"{float(mem_total)/1024:.1f}G"})
|
||||
return gpus or "未检测到 GPU"
|
||||
except FileNotFoundError:
|
||||
return "不可用(容器内无 nvidia-smi,需 compose 配置 GPU)"
|
||||
except Exception as err: # noqa: BLE001
|
||||
return f"检查失败: {str(err)[:80]}"
|
||||
|
||||
|
||||
def _inspect_container(client: httpx.Client, cid: str) -> dict:
|
||||
try:
|
||||
resp = client.get(f"http://localhost/containers/{cid}/json", timeout=10)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except Exception: # noqa: BLE001
|
||||
return {}
|
||||
|
||||
|
||||
def _check_docker() -> dict:
|
||||
"""容器状态汇总 + 异常容器列表(应运行但退出/restarting/unhealthy/OOMKilled)。"""
|
||||
if not os.path.exists(DOCKER_SOCK):
|
||||
return {"running": None, "issues": [{"name": "Docker", "issue": f"不可用(未挂载 {DOCKER_SOCK})"}]}
|
||||
with httpx.Client(transport=httpx.HTTPTransport(uds=DOCKER_SOCK), timeout=10) as client:
|
||||
resp = client.get("http://localhost/containers/json", params={"all": "1"})
|
||||
resp.raise_for_status()
|
||||
containers = resp.json()
|
||||
running = sum(1 for c in containers if c.get("State") == "running")
|
||||
issues = []
|
||||
stopped = 0
|
||||
for c in containers:
|
||||
name = (c.get("Names") or ["?"])[0].lstrip("/")
|
||||
state, status = c.get("State", ""), c.get("Status", "")
|
||||
if state == "restarting":
|
||||
issues.append({"name": name, "issue": f"反复重启 ({status})"})
|
||||
elif state == "running" and "(unhealthy)" in status:
|
||||
issues.append({"name": name, "issue": f"健康检查失败 ({status})"})
|
||||
elif state in ("exited", "dead"):
|
||||
detail = _inspect_container(client, c["Id"])
|
||||
policy = ((detail.get("HostConfig") or {}).get("RestartPolicy") or {}).get("Name", "no")
|
||||
oom = (detail.get("State") or {}).get("OOMKilled", False)
|
||||
finished_at = (detail.get("State") or {}).get("FinishedAt", "")
|
||||
if oom and _exited_recently(finished_at):
|
||||
issues.append({"name": name, "issue": f"被 OOM 杀死 ({status})"})
|
||||
elif policy in ("always", "unless-stopped"):
|
||||
if _exited_recently(finished_at):
|
||||
issues.append({"name": name, "issue": f"应常驻但刚退出 ({status}, restart={policy})"})
|
||||
else:
|
||||
stopped += 1 # 数周前人为停用的旧服务,不算异常
|
||||
return {"running": running, "total": len(containers), "issues": issues, "stopped": stopped}
|
||||
|
||||
|
||||
def _exited_recently(finished_at: str, hours: int = 24) -> bool:
|
||||
try:
|
||||
from datetime import timezone
|
||||
dt = datetime.fromisoformat(finished_at.replace("Z", "+00:00"))
|
||||
return (datetime.now(timezone.utc) - dt).total_seconds() < hours * 3600
|
||||
except (ValueError, TypeError):
|
||||
return True # 解析失败按新退出处理,宁可多报
|
||||
|
||||
|
||||
def check_prefect_failures(hours: int = 24) -> list[dict]:
|
||||
api_url = os.environ.get("PREFECT_API_URL", "http://localhost:4200/api")
|
||||
since = (datetime.now().astimezone() - timedelta(hours=hours)).isoformat()
|
||||
resp = httpx.post(
|
||||
f"{api_url}/flow_runs/filter",
|
||||
json={
|
||||
"flow_runs": {"state": {"type": {"any_": ["FAILED", "CRASHED"]}}, "start_time": {"after_": since}},
|
||||
"limit": 50,
|
||||
"sort": "START_TIME_DESC",
|
||||
},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
out = []
|
||||
for r in resp.json():
|
||||
start = (r.get("start_time") or "")[:16].replace("T", " ")
|
||||
out.append({"name": r.get("name", "?"), "state": (r.get("state") or {}).get("type", "?"),
|
||||
"message": ((r.get("state") or {}).get("message") or "").strip()[:100], "start": start})
|
||||
return out
|
||||
|
||||
|
||||
def _check_services() -> dict:
|
||||
from api.ocr import OcrClient
|
||||
|
||||
ocr_ok = False
|
||||
try:
|
||||
ocr_ok = OcrClient().health()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
pg_ok = False
|
||||
try:
|
||||
pg_ok = bool(SyncDB().ping())
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return {"ocr": ocr_ok, "pg": pg_ok}
|
||||
|
||||
|
||||
def _mark(pct: float | None, threshold: int) -> str:
|
||||
if pct is None:
|
||||
return "—"
|
||||
return f"{pct:.0f}%" + (" 🔴" if pct >= threshold else "")
|
||||
|
||||
|
||||
@flow(name="device-check", log_prints=True)
|
||||
def device_check(alarm_threshold: int = 80) -> dict:
|
||||
"""设备/系统健康日报:检查并直接发企微群。单项检查失败不影响其余项。"""
|
||||
checks: dict[str, object] = {}
|
||||
for name, fn in [
|
||||
("memory", _check_memory), ("disks", _check_disks), ("gpus", _check_gpus),
|
||||
("docker", _check_docker), ("prefectFailures", check_prefect_failures), ("services", _check_services),
|
||||
]:
|
||||
try:
|
||||
checks[name] = fn()
|
||||
except Exception as err: # noqa: BLE001
|
||||
checks[name] = None
|
||||
print(f"[device-check] {name} 检查失败: {err}")
|
||||
|
||||
threshold = alarm_threshold
|
||||
lines = [f"🖥 **设备健康日报** {datetime.now().strftime('%Y-%m-%d %H:%M')}", ""]
|
||||
|
||||
mem = checks.get("memory") or {}
|
||||
if mem:
|
||||
lines.append(f"**内存**: {_mark(mem['pct'], threshold)}({mem['used']}/{mem['total']})"
|
||||
f" | swap {_mark(mem['swapPct'], threshold)}({mem['swapUsed']}/{mem['swapTotal']})")
|
||||
else:
|
||||
lines.append("**内存**: 检查失败")
|
||||
|
||||
lines.append("**磁盘**:")
|
||||
lines.append("| 挂载点 | 使用率 | 已用/总量 |")
|
||||
lines.append("| --- | --- | --- |")
|
||||
for d in checks.get("disks") or []:
|
||||
if "error" in d:
|
||||
lines.append(f"| {d['label']} | 检查失败 | {d['error']} |")
|
||||
else:
|
||||
lines.append(f"| {d['label']} | {_mark(d['pct'], threshold)} | {d['used']}/{d['total']} |")
|
||||
|
||||
lines.append("**GPU**:")
|
||||
gpus = checks.get("gpus")
|
||||
if isinstance(gpus, list) and gpus:
|
||||
lines.append("| GPU | 利用率 | 显存 |")
|
||||
lines.append("| --- | --- | --- |")
|
||||
for g in gpus:
|
||||
lines.append(f"| {g['name']} | {_mark(g['utilPct'], threshold)} | "
|
||||
f"{_mark(g['memPct'], threshold)}({g['memUsed']}/{g['memTotal']}) |")
|
||||
else:
|
||||
lines.append(f"> {gpus if isinstance(gpus, str) else '未检测到'}")
|
||||
|
||||
docker = checks.get("docker") or {}
|
||||
issues = docker.get("issues") or []
|
||||
if docker.get("running") is None:
|
||||
lines.append(f"**Docker**: {issues[0]['issue'] if issues else '不可用'}")
|
||||
else:
|
||||
lines.append(f"**Docker 容器**: 运行中 {docker['running']}/{docker.get('total', '?')}"
|
||||
+ (f",已停用 {docker['stopped']}" if docker.get("stopped") else "")
|
||||
+ (f",⚠️ 异常 {len(issues)} 个" if issues else ",全部正常"))
|
||||
for it in issues[:10]:
|
||||
lines.append(f"- 🔴 {it['name']}: {it['issue']}")
|
||||
|
||||
failures = checks.get("prefectFailures") or []
|
||||
lines.append(f"**Prefect 近24h 失败**: {len(failures)} 个 flow run"
|
||||
+ ("" if failures else " ✅"))
|
||||
for f in failures[:8]:
|
||||
lines.append(f"- 🔴 {f['name']}({f['state']}) {f['start']} {f['message'][:60]}")
|
||||
|
||||
svc = checks.get("services") or {}
|
||||
lines.append(f"**服务探活**: OCR {'✅' if svc.get('ocr') else '🔴'} | "
|
||||
f"PG {'✅' if svc.get('pg') else '🔴'} | 负载 {os.getloadavg()[0]:.1f}")
|
||||
|
||||
alarms = (
|
||||
(mem and (mem["pct"] >= threshold or mem["swapPct"] >= threshold))
|
||||
or any(d.get("pct", 0) >= threshold for d in (checks.get("disks") or []))
|
||||
or (isinstance(gpus, list) and any(g["utilPct"] >= threshold or g["memPct"] >= threshold for g in gpus))
|
||||
or bool(issues) or bool(failures) or not svc.get("ocr") or not svc.get("pg")
|
||||
)
|
||||
lines += ["", ("🔴 **有项目需要处理,见上**" if alarms else "✅ **一切正常**")]
|
||||
|
||||
content = "\n".join(lines)
|
||||
sent = send_markdown(content)
|
||||
emit_sync_summary("设备健康日报", [
|
||||
("内存", f"{_mark(mem['pct'], threshold)}" if mem else "—"),
|
||||
("Docker 异常", len(issues)),
|
||||
("Prefect 失败(24h)", len(failures)),
|
||||
("OCR", "✅" if svc.get("ocr") else "🔴"), ("PG", "✅" if svc.get("pg") else "🔴"),
|
||||
("已发企微", "是" if sent else "否"),
|
||||
])
|
||||
return {"alarms": bool(alarms), "sent": sent, "dockerIssues": len(issues), "prefectFailures": len(failures)}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
device_check.serve(name="device-check", cron="0 9 * * *")
|
||||
Reference in New Issue
Block a user