feat(cli): base/space 命令 + 执行守则 + eval harness 入仓 + 多轮 eval (0.1.9)

- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖) - SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引 (snapshot 无恢复 API,用 deactivate+节点改回) - hint 补 429 限流与超时模式 - evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/ EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分 - 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用 - 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次; 多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话) - 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
2026-09-30 08:12:16 +00:00
parent 517dde9dee
commit c9694511a1
20 changed files with 1607 additions and 9 deletions
@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Launch inkreach-teable-plugin skill evals (integrity-clean port).
+
+Usage: python3 run_plugin_evals.py <iteration> <tag> <config-spec> [eval-ids]
+  config-spec: with_skill=/root/eval-rerun/skill-plugin-new:/root/eval-rerun/bin-new/bin
+"""
+import json, os, re, shutil, subprocess, sys, time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+HARNESS = Path(__file__).parent  # evals/ itself
+ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
+MODEL = os.environ.get("EVAL_MODEL", "opencode-go/deepseek-v4.1-flash")
+RUN_TIMEOUT = 900  # plugin evals build bundles — need more headroom
+PI_HOME = os.environ.get("EVAL_PI_HOME", "/root/eval-rerun/pi-home")
+FIXTURE = "/root/pi-skill-lab/plugin-workspace/fixture-deploy-W"
+
+env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'plugin-sandbox-env.json'))).read_text())
+
+PROMPT_TMPL = """你在为一套已部署的 inkreach-teable 实例(Teable fork)开发/部署动态 React 插件。
+
+环境(已配置好,直接使用):
+- 后端 http://localhost:3003,前端 http://localhost:3002。PAT 已通过环境变量 INKREACH_TEABLE_ENDPOINT / INKREACH_TEABLE_TOKEN 注入(`inkreach-teable` CLI 可直接用)。
+- 评估沙箱基地 ID:{BASE}。凡涉及实例的读写一律只在这个基地内,严禁触碰其他基地;严禁向实例的插件注册表上传与任务无关的演示插件。
+- 工作目录就是当前目录,所有产出写在这里。
+
+{SKILL_LINE}
+
+任务:{TASK}
+
+要求:
+1. 按任务性质完成:咨询类问题(能不能/怎么做)直接给出方案与关键代码即可,不必搭建完整工程;开发/部署类任务才需要实际构建与部署。
+2. 完成后把最终答复写入 final_answer.md:结论、关键实现/命令、结果、需要用户知道的事项。"""
+
+
+def extract_final_text(p: Path) -> str:
+    texts = []
+    for line in p.read_text(errors="replace").splitlines():
+        try:
+            ev = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if ev.get("type") == "turn_end" and ev.get("message", {}).get("role") == "assistant":
+            for c in ev["message"].get("content", []):
+                if c.get("type") == "text" and c.get("text", "").strip():
+                    texts.append(c["text"])
+    return texts[-1] if texts else ""
+
+
+def run_one(task, config, skill_path, bin_dir, cfg_env, run_dir: Path):
+    run_dir.mkdir(parents=True, exist_ok=True)
+    skill_line = (
+        f"- 先完整阅读技能说明 {skill_path}/SKILL.md 并按其模块地图(按需读 dynamic-plugins/ 下文档)操作。"
+        if skill_path else ""
+    )
+    prompt = PROMPT_TMPL.format(BASE=cfg_env["INKREACH_TEABLE_BASE_ID"], SKILL_LINE=skill_line, TASK=task)
+    (run_dir / "prompt.txt").write_text(prompt)
+    full_env = {
+        **os.environ, "HOME": PI_HOME, "PATH": f"{bin_dir}:{os.environ['PATH']}",
+        **{k: env_all[k] for k in ("INKREACH_TEABLE_ENDPOINT", "INKREACH_TEABLE_TOKEN")},
+        **cfg_env,
+    }
+    cmd = ["timeout", str(RUN_TIMEOUT), "pi", "-p", "--mode", "json", "--model", MODEL, prompt]
+    t0 = time.time()
+    with open(run_dir / "pi.jsonl", "w") as out:
+        proc = subprocess.run(cmd, stdout=out, stderr=subprocess.DEVNULL, cwd=run_dir, env=full_env)
+    (run_dir / "timing.json").write_text(json.dumps(
+        {"duration_s": round(time.time() - t0, 1), "exit_code": proc.returncode, "config": config}))
+    (run_dir / "assistant_final.md").write_text(extract_final_text(run_dir / "pi.jsonl") or "(none)")
+    return config, run_dir.name, proc.returncode
+
+
+def main():
+    iteration, tag = sys.argv[1], sys.argv[2]
+    iteration_dir = ROOT / iteration
+    configs = {}
+    for part in sys.argv[3].split(","):
+        name, _, rest = part.partition("=")
+        sp, _, bd = rest.partition(":")
+        configs[name] = (sp, bd)
+    evals = json.loads(Path(os.environ.get('EVAL_EVALS', str(HARNESS / 'plugin-evals.json'))).read_text())["evals"]
+    if len(sys.argv) > 4 and sys.argv[4]:
+        only = {int(x) for x in sys.argv[4].split(",")}
+        evals = [e for e in evals if e["id"] in only]
+
+    jobs = []
+    for ev in evals:
+        name = f"eval-{ev['id']}-{ev['name']}"
+        for config, (skill_path, bin_dir) in configs.items():
+            cfg = env_all[config]
+            cfg_short = "W" if config == "with_skill" else "O"
+            task = ev["prompt"].replace("{TBL}", cfg["TBL"])
+            rd = iteration_dir / name / config / "run-1"
+            rd.parent.mkdir(parents=True, exist_ok=True)
+            # eval-3 needs the ready-plugin fixture bundle in cwd
+            if ev["id"] == 3:
+                rp = rd / "ready-plugin"
+                if rp.exists():
+                    shutil.rmtree(rp)
+                shutil.copytree(FIXTURE, rp)
+            (rd.parent / "eval_metadata.json").write_text(json.dumps(
+                {"eval_id": ev["id"], "prompt": task, "assertions": []}, ensure_ascii=False))
+            cfg_env = {"INKREACH_TEABLE_BASE_ID": cfg["INKREACH_TEABLE_BASE_ID"]}
+            jobs.append((task, config, skill_path, bin_dir, cfg_env, rd))
+
+    print(f"{len(jobs)} runs -> {iteration_dir}", flush=True)
+    with ThreadPoolExecutor(max_workers=3) as ex:  # builds are heavy — 3 workers
+        for f in [ex.submit(run_one, *j) for j in jobs]:
+            print(f.result(), flush=True)
+
+
+if __name__ == "__main__":
+    main()