feat(cli): base/space 命令 + 执行守则 + eval harness 入仓 + 多轮 eval (0.1.9)
- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖) - SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引 (snapshot 无恢复 API,用 deactivate+节点改回) - hint 补 429 限流与超时模式 - evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/ EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分 - 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用 - 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次; 多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话) - 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
This commit is contained in:
yeuimu
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade plugin-skill eval runs (ported from plugin-workspace, per-config sandboxes)."""
|
||||
import json, os, re, subprocess, sys
|
||||
from pathlib import Path
|
||||
|
||||
HARNESS = Path(__file__).parent # evals/ itself
|
||||
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
|
||||
CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable")
|
||||
env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'plugin-sandbox-env.json'))).read_text())
|
||||
|
||||
FORBIDDEN_IMPORT = re.compile(r"^\s*import\s+(?!type)[^\n]*from\s+['\"]@teable/(sdk|ui-lib|openapi|core|icons)", re.M)
|
||||
REACT_IMPORT = re.compile(r"^\s*import\s+(?!type)[^\n]*from\s+['\"]react['\"]", re.M)
|
||||
|
||||
|
||||
def cli_json(*args):
|
||||
e = {**os.environ,
|
||||
"INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
|
||||
"INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
|
||||
"INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"]}
|
||||
p = subprocess.run([CLI, *args, "--json"], capture_output=True, text=True, env=e, timeout=60)
|
||||
try:
|
||||
return json.loads(p.stdout)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def transcript(run_dir: Path):
|
||||
cmds = []
|
||||
for line in (run_dir / "pi.jsonl").read_text(errors="replace").splitlines():
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if ev.get("type") == "tool_execution_start" and ev.get("toolName") == "bash":
|
||||
cmds.append(ev.get("args", {}).get("command", ""))
|
||||
return cmds
|
||||
|
||||
|
||||
def answer(run_dir: Path) -> str:
|
||||
fa, af = run_dir / "final_answer.md", run_dir / "assistant_final.md"
|
||||
return fa.read_text(errors="replace") if fa.exists() else af.read_text(errors="replace")
|
||||
|
||||
|
||||
def plugin_sources(run_dir: Path):
|
||||
files = sorted((run_dir / "plugin").rglob("*.[tj]s*")) or sorted(run_dir.rglob("plugin.json"))
|
||||
return "\n".join(p.read_text(errors="replace") for p in files), files
|
||||
|
||||
|
||||
def result(text, passed, evidence):
|
||||
return {"text": text, "passed": bool(passed), "evidence": str(evidence)[:400]}
|
||||
|
||||
|
||||
def grade_eval(ev_name, run_dir: Path, cfg_short: str):
|
||||
cmds = transcript(run_dir)
|
||||
ans = answer(run_dir)
|
||||
out = []
|
||||
|
||||
if ev_name in ("contextmenu-plugin", "panel-widget"):
|
||||
src, files = plugin_sources(run_dir)
|
||||
want_pos = "contextMenu" if ev_name == "contextmenu-plugin" else "panel"
|
||||
want_api = "addContextMenuAction" if ev_name == "contextmenu-plugin" else "addPanel"
|
||||
pj = run_dir / "plugin" / "plugin.json"
|
||||
manifest_ok = False
|
||||
if pj.exists():
|
||||
try:
|
||||
m = json.loads(pj.read_text())
|
||||
manifest_ok = bool(re.fullmatch(r"[a-z0-9-]+", m.get("id", ""))) and want_pos in (m.get("positions") or [])
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
out.append(result(f"plugin.json 存在且 manifest 合法(id 合规、positions 含 {want_pos})", manifest_ok,
|
||||
pj.read_text(errors="replace")[:150] if pj.exists() else "plugin/plugin.json 未找到"))
|
||||
out.append(result("代码从 window.__TEABLE_HOST__ 获取宿主能力", "__TEABLE_HOST__" in src,
|
||||
"host global: %s" % ("有" if "__TEABLE_HOST__" in src else "无")))
|
||||
bad = FORBIDDEN_IMPORT.findall(src)
|
||||
out.append(result("@teable/* 无运行时值导入(仅允许 import type)", not bad, "违规: %r" % bad if bad else "ok"))
|
||||
uses_react = bool(REACT_IMPORT.search(src))
|
||||
wp = run_dir / "plugin" / "webpack.config.js"
|
||||
wp_txt = wp.read_text(errors="replace") if wp.exists() else ""
|
||||
externals_ok = "__TEABLE_HOST__" in wp_txt and "externals" in wp_txt
|
||||
out.append(result("react 导入经 externals 映射(或无 react 裸导入)", externals_ok if uses_react else True,
|
||||
"webpack externals: %s" % ("映射" if externals_ok else ("未映射!" if uses_react else "不需要"))))
|
||||
out.append(result(f"definePlugin 注册且调用 ctx.{want_api}", "definePlugin" in src and want_api in src,
|
||||
"definePlugin=%s %s=%s" % ("definePlugin" in src, want_api, want_api in src)))
|
||||
if ev_name == "contextmenu-plugin":
|
||||
edits = re.search(r"updateRecords|record.*update|OpenAPI", src, re.I)
|
||||
out.append(result("实现了批量更新记录逻辑", bool(edits), edits.group(0) if edits else "未发现"))
|
||||
else:
|
||||
sel = re.search(r"useSelection|selection|getSelectionRecords", src, re.I)
|
||||
out.append(result("通过宿主 SDK 获取选中记录", bool(sel), sel.group(0) if sel else "未发现"))
|
||||
|
||||
elif ev_name == "deploy-chain":
|
||||
plugin_id = f"eval-deploy-{cfg_short}"
|
||||
up = any("upload-bundle" in c for c in cmds)
|
||||
out.append(result("调用了 upload-bundle 上传", up, [c for c in cmds if "upload-bundle" in c][:1]))
|
||||
enabled = any("plugin-config" in c for c in cmds)
|
||||
out.append(result("启用了 plugin-config", enabled, [c for c in cmds if "plugin-config" in c][:1]))
|
||||
installed = any(re.search(r"context-menu.*install|plugin-context-menu", c) for c in cmds)
|
||||
out.append(result("安装到表级 context-menu 位置", installed, [c for c in cmds if "context-menu" in c][:1]))
|
||||
cleaned = any(("DELETE" in c.upper() and "/api/plugin/" in c) or re.search(r"plugin uninstall|plugin-install", c) for c in cmds) \
|
||||
or (cli_json("plugin", "list") or [] and not any(p.get("id") == plugin_id for p in (cli_json("plugin", "list") or [])) and any("install" in c for c in cmds))
|
||||
out.append(result("执行了清理(卸载安装/删除插件)", cleaned, [c for c in cmds if "uninstall" in c or "DELETE" in c.upper()][:2]))
|
||||
lst = json.dumps(cli_json("plugin", "list") or [], ensure_ascii=False)
|
||||
out.append(result("实例上无插件残留(plugin list 不含该 id)", plugin_id not in lst,
|
||||
"残留!" if plugin_id in lst else "干净"))
|
||||
verify = re.search(r"已确认|确认(安装|清理|删除)|installed|verified|清理(完成|干净)", ans)
|
||||
out.append(result("答复确认了安装与清理结果", bool(verify), ans[:200]))
|
||||
|
||||
elif ev_name == "proxy-rule":
|
||||
out.append(result("方案使用同源代理 /api/plugin-proxy/<service>/...", "/api/plugin-proxy/" in ans, ans[:250]))
|
||||
out.append(result("指出需注册 PLUGIN_API_SERVICES 并重启 teable-backend", "PLUGIN_API_SERVICES" in ans,
|
||||
"提及" if "PLUGIN_API_SERVICES" in ans else "未提及"))
|
||||
lan_fetch = re.search(r"fetch\(\s*['\"]http://192\.168", ans)
|
||||
out.append(result("不建议在 bundle 里直接请求局域网 IP", lan_fetch is None, lan_fetch.group(0) if lan_fetch else "ok"))
|
||||
|
||||
elif ev_name == "pitfall-diagnosis":
|
||||
cause = re.search(r"(@teable/sdk|值导入|运行时导入|externals|undefined)|import", ans) and re.search(r"__TEABLE_HOST__|宿主全局|host global", ans)
|
||||
out.append(result("定位为 @teable/* 运行时导入被 externals 映射为 undefined", bool(cause), ans[:250]))
|
||||
out.append(result("给出修复:改从 window.__TEABLE_HOST__ 读取(或仅 import type)", "__TEABLE_HOST__" in ans, ans[:200]))
|
||||
|
||||
elif ev_name == "native-form-routing":
|
||||
nf = re.search(r"NativeForm|useNativeForm|NativeFormScope", ans)
|
||||
out.append(result("指出原生表单注入能力(NativeForm/NativeFormScope/useNativeForm)", bool(nf),
|
||||
nf.group(0) if nf else ans[:200]))
|
||||
# 咨询类任务应给出可实现方案(两档之一 + 关键代码/props)
|
||||
plan = bool(nf) and bool(re.search(r"visibleFieldIds|slots|NativeFormField|formId|interceptSubmit", ans))
|
||||
out.append(result("给出了具体实现要点(props/多步结构)", plan, ans[:250]))
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
iteration_dir = ROOT / sys.argv[1]
|
||||
for eval_dir in sorted(iteration_dir.glob("eval-*")):
|
||||
ev_name = eval_dir.name.split("-", 2)[2]
|
||||
for config_dir in sorted(p for p in eval_dir.iterdir() if p.is_dir()):
|
||||
run_dir = config_dir / "run-1"
|
||||
if not (run_dir / "pi.jsonl").exists():
|
||||
continue
|
||||
cfg_short = "W" if config_dir.name == "with_skill" else "O"
|
||||
try:
|
||||
expectations = grade_eval(ev_name, run_dir, cfg_short)
|
||||
except Exception as e:
|
||||
expectations = [result("判分异常", False, repr(e))]
|
||||
(run_dir / "grading.json").write_text(json.dumps({"expectations": expectations}, indent=2, ensure_ascii=False))
|
||||
npass = sum(1 for x in expectations if x["passed"])
|
||||
print(f"{eval_dir.name:34} {config_dir.name:12} {npass}/{len(expectations)}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user