yeuimu
c9694511a1
- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖) - SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引 (snapshot 无恢复 API,用 deactivate+节点改回) - hint 补 429 限流与超时模式 - evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/ EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分 - 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用 - 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次; 多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话) - 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
151 lines
8.6 KiB
Python
151 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade plugin-skill eval runs (ported from plugin-workspace, per-config sandboxes)."""
|
|
import json, os, re, subprocess, sys
|
|
from pathlib import Path
|
|
|
|
HARNESS = Path(__file__).parent # evals/ itself
|
|
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
|
|
CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable")
|
|
env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'plugin-sandbox-env.json'))).read_text())
|
|
|
|
FORBIDDEN_IMPORT = re.compile(r"^\s*import\s+(?!type)[^\n]*from\s+['\"]@teable/(sdk|ui-lib|openapi|core|icons)", re.M)
|
|
REACT_IMPORT = re.compile(r"^\s*import\s+(?!type)[^\n]*from\s+['\"]react['\"]", re.M)
|
|
|
|
|
|
def cli_json(*args):
|
|
e = {**os.environ,
|
|
"INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
|
|
"INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
|
|
"INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"]}
|
|
p = subprocess.run([CLI, *args, "--json"], capture_output=True, text=True, env=e, timeout=60)
|
|
try:
|
|
return json.loads(p.stdout)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
|
|
|
|
def transcript(run_dir: Path):
|
|
cmds = []
|
|
for line in (run_dir / "pi.jsonl").read_text(errors="replace").splitlines():
|
|
try:
|
|
ev = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if ev.get("type") == "tool_execution_start" and ev.get("toolName") == "bash":
|
|
cmds.append(ev.get("args", {}).get("command", ""))
|
|
return cmds
|
|
|
|
|
|
def answer(run_dir: Path) -> str:
|
|
fa, af = run_dir / "final_answer.md", run_dir / "assistant_final.md"
|
|
return fa.read_text(errors="replace") if fa.exists() else af.read_text(errors="replace")
|
|
|
|
|
|
def plugin_sources(run_dir: Path):
|
|
files = sorted((run_dir / "plugin").rglob("*.[tj]s*")) or sorted(run_dir.rglob("plugin.json"))
|
|
return "\n".join(p.read_text(errors="replace") for p in files), files
|
|
|
|
|
|
def result(text, passed, evidence):
|
|
return {"text": text, "passed": bool(passed), "evidence": str(evidence)[:400]}
|
|
|
|
|
|
def grade_eval(ev_name, run_dir: Path, cfg_short: str):
|
|
cmds = transcript(run_dir)
|
|
ans = answer(run_dir)
|
|
out = []
|
|
|
|
if ev_name in ("contextmenu-plugin", "panel-widget"):
|
|
src, files = plugin_sources(run_dir)
|
|
want_pos = "contextMenu" if ev_name == "contextmenu-plugin" else "panel"
|
|
want_api = "addContextMenuAction" if ev_name == "contextmenu-plugin" else "addPanel"
|
|
pj = run_dir / "plugin" / "plugin.json"
|
|
manifest_ok = False
|
|
if pj.exists():
|
|
try:
|
|
m = json.loads(pj.read_text())
|
|
manifest_ok = bool(re.fullmatch(r"[a-z0-9-]+", m.get("id", ""))) and want_pos in (m.get("positions") or [])
|
|
except json.JSONDecodeError:
|
|
pass
|
|
out.append(result(f"plugin.json 存在且 manifest 合法(id 合规、positions 含 {want_pos})", manifest_ok,
|
|
pj.read_text(errors="replace")[:150] if pj.exists() else "plugin/plugin.json 未找到"))
|
|
out.append(result("代码从 window.__TEABLE_HOST__ 获取宿主能力", "__TEABLE_HOST__" in src,
|
|
"host global: %s" % ("有" if "__TEABLE_HOST__" in src else "无")))
|
|
bad = FORBIDDEN_IMPORT.findall(src)
|
|
out.append(result("@teable/* 无运行时值导入(仅允许 import type)", not bad, "违规: %r" % bad if bad else "ok"))
|
|
uses_react = bool(REACT_IMPORT.search(src))
|
|
wp = run_dir / "plugin" / "webpack.config.js"
|
|
wp_txt = wp.read_text(errors="replace") if wp.exists() else ""
|
|
externals_ok = "__TEABLE_HOST__" in wp_txt and "externals" in wp_txt
|
|
out.append(result("react 导入经 externals 映射(或无 react 裸导入)", externals_ok if uses_react else True,
|
|
"webpack externals: %s" % ("映射" if externals_ok else ("未映射!" if uses_react else "不需要"))))
|
|
out.append(result(f"definePlugin 注册且调用 ctx.{want_api}", "definePlugin" in src and want_api in src,
|
|
"definePlugin=%s %s=%s" % ("definePlugin" in src, want_api, want_api in src)))
|
|
if ev_name == "contextmenu-plugin":
|
|
edits = re.search(r"updateRecords|record.*update|OpenAPI", src, re.I)
|
|
out.append(result("实现了批量更新记录逻辑", bool(edits), edits.group(0) if edits else "未发现"))
|
|
else:
|
|
sel = re.search(r"useSelection|selection|getSelectionRecords", src, re.I)
|
|
out.append(result("通过宿主 SDK 获取选中记录", bool(sel), sel.group(0) if sel else "未发现"))
|
|
|
|
elif ev_name == "deploy-chain":
|
|
plugin_id = f"eval-deploy-{cfg_short}"
|
|
up = any("upload-bundle" in c for c in cmds)
|
|
out.append(result("调用了 upload-bundle 上传", up, [c for c in cmds if "upload-bundle" in c][:1]))
|
|
enabled = any("plugin-config" in c for c in cmds)
|
|
out.append(result("启用了 plugin-config", enabled, [c for c in cmds if "plugin-config" in c][:1]))
|
|
installed = any(re.search(r"context-menu.*install|plugin-context-menu", c) for c in cmds)
|
|
out.append(result("安装到表级 context-menu 位置", installed, [c for c in cmds if "context-menu" in c][:1]))
|
|
cleaned = any(("DELETE" in c.upper() and "/api/plugin/" in c) or re.search(r"plugin uninstall|plugin-install", c) for c in cmds) \
|
|
or (cli_json("plugin", "list") or [] and not any(p.get("id") == plugin_id for p in (cli_json("plugin", "list") or [])) and any("install" in c for c in cmds))
|
|
out.append(result("执行了清理(卸载安装/删除插件)", cleaned, [c for c in cmds if "uninstall" in c or "DELETE" in c.upper()][:2]))
|
|
lst = json.dumps(cli_json("plugin", "list") or [], ensure_ascii=False)
|
|
out.append(result("实例上无插件残留(plugin list 不含该 id)", plugin_id not in lst,
|
|
"残留!" if plugin_id in lst else "干净"))
|
|
verify = re.search(r"已确认|确认(安装|清理|删除)|installed|verified|清理(完成|干净)", ans)
|
|
out.append(result("答复确认了安装与清理结果", bool(verify), ans[:200]))
|
|
|
|
elif ev_name == "proxy-rule":
|
|
out.append(result("方案使用同源代理 /api/plugin-proxy/<service>/...", "/api/plugin-proxy/" in ans, ans[:250]))
|
|
out.append(result("指出需注册 PLUGIN_API_SERVICES 并重启 teable-backend", "PLUGIN_API_SERVICES" in ans,
|
|
"提及" if "PLUGIN_API_SERVICES" in ans else "未提及"))
|
|
lan_fetch = re.search(r"fetch\(\s*['\"]http://192\.168", ans)
|
|
out.append(result("不建议在 bundle 里直接请求局域网 IP", lan_fetch is None, lan_fetch.group(0) if lan_fetch else "ok"))
|
|
|
|
elif ev_name == "pitfall-diagnosis":
|
|
cause = re.search(r"(@teable/sdk|值导入|运行时导入|externals|undefined)|import", ans) and re.search(r"__TEABLE_HOST__|宿主全局|host global", ans)
|
|
out.append(result("定位为 @teable/* 运行时导入被 externals 映射为 undefined", bool(cause), ans[:250]))
|
|
out.append(result("给出修复:改从 window.__TEABLE_HOST__ 读取(或仅 import type)", "__TEABLE_HOST__" in ans, ans[:200]))
|
|
|
|
elif ev_name == "native-form-routing":
|
|
nf = re.search(r"NativeForm|useNativeForm|NativeFormScope", ans)
|
|
out.append(result("指出原生表单注入能力(NativeForm/NativeFormScope/useNativeForm)", bool(nf),
|
|
nf.group(0) if nf else ans[:200]))
|
|
# 咨询类任务应给出可实现方案(两档之一 + 关键代码/props)
|
|
plan = bool(nf) and bool(re.search(r"visibleFieldIds|slots|NativeFormField|formId|interceptSubmit", ans))
|
|
out.append(result("给出了具体实现要点(props/多步结构)", plan, ans[:250]))
|
|
|
|
return out
|
|
|
|
|
|
def main():
|
|
iteration_dir = ROOT / sys.argv[1]
|
|
for eval_dir in sorted(iteration_dir.glob("eval-*")):
|
|
ev_name = eval_dir.name.split("-", 2)[2]
|
|
for config_dir in sorted(p for p in eval_dir.iterdir() if p.is_dir()):
|
|
run_dir = config_dir / "run-1"
|
|
if not (run_dir / "pi.jsonl").exists():
|
|
continue
|
|
cfg_short = "W" if config_dir.name == "with_skill" else "O"
|
|
try:
|
|
expectations = grade_eval(ev_name, run_dir, cfg_short)
|
|
except Exception as e:
|
|
expectations = [result("判分异常", False, repr(e))]
|
|
(run_dir / "grading.json").write_text(json.dumps({"expectations": expectations}, indent=2, ensure_ascii=False))
|
|
npass = sum(1 for x in expectations if x["passed"])
|
|
print(f"{eval_dir.name:34} {config_dir.name:12} {npass}/{len(expectations)}", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|