feat(cli): base/space 命令 + 执行守则 + eval harness 入仓 + 多轮 eval (0.1.9)
- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖) - SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引 (snapshot 无恢复 API,用 deactivate+节点改回) - hint 补 429 限流与超时模式 - evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/ EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分 - 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用 - 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次; 多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话) - 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Multi-turn evals: two-round conversations where turn 2 corrects turn 1.
|
||||
|
||||
Verifies the agent AMENDS the previous artifacts (edit workflow / add field)
|
||||
instead of ignoring the correction or rebuilding blindly.
|
||||
|
||||
Usage: python3 run_multiturn.py <iteration> <tag>
|
||||
"""
|
||||
import json, os, subprocess, sys, time
|
||||
from pathlib import Path
|
||||
|
||||
HARNESS = Path(__file__).parent
|
||||
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
|
||||
MODEL = os.environ.get("EVAL_MODEL", "opencode-go/deepseek-v4.1-flash")
|
||||
RUN_TIMEOUT = 480
|
||||
PI_HOME = os.environ.get("EVAL_PI_HOME_SKILLED", "/root/eval-rerun/pi-home-skilled") # trigger mode: skill auto-discovered
|
||||
|
||||
env_all = json.loads((HARNESS / "sandbox-env.json").read_text())
|
||||
|
||||
TASKS = [
|
||||
{
|
||||
"name": "mt-1-notify-correction",
|
||||
"turn1": "帮我建个自动化:order_logs 表有新记录的时候通知我。名字你来定,配好激活。",
|
||||
"turn2": "不对,改一下:别用邮件,要站内通知;而且只在「结果」字段变成「失败」的时候才通知,不是有新记录就通知。",
|
||||
"turn1_hint": None,
|
||||
},
|
||||
{
|
||||
"name": "mt-2-incremental-schema",
|
||||
"turn1": "帮我建张表存客户反馈,加个单行文本「内容」字段,随便插一条示例。",
|
||||
"turn2": "再补一个单选字段「渠道」,选项:微信/电话/邮件。然后把刚才那条示例记录的渠道标成「微信」。",
|
||||
"turn1_hint": None,
|
||||
},
|
||||
]
|
||||
|
||||
WRAP = """要求:
|
||||
1. 逐步用 CLI 完成实际操作(不是只给命令)。
|
||||
2. 每轮结束后把当前轮的答复写入 final_answer_turn{N}.md(做了什么、关键命令、结果)。
|
||||
3. 用户的后续修正优先于之前的做法;能改就改现有的,不要无视."""
|
||||
|
||||
|
||||
def turn(run_dir: Path, sid: str, text: str, n: int):
|
||||
prompt = f"{text}\n\n{WRAP.format(N=n)}"
|
||||
cmd = ["timeout", str(RUN_TIMEOUT), "pi", "-p", "--mode", "json",
|
||||
"--session-id", sid, "--model", MODEL, prompt]
|
||||
out_f = run_dir / f"turn{n}.jsonl"
|
||||
env = {
|
||||
**os.environ, "HOME": PI_HOME,
|
||||
"PATH": f"/root/eval-rerun/bin-new/bin:{os.environ['PATH']}",
|
||||
"INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
|
||||
"INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
|
||||
"INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"],
|
||||
}
|
||||
with open(out_f, "w") as out:
|
||||
p = subprocess.run(cmd, stdout=out, stderr=subprocess.DEVNULL, cwd=run_dir, env=env)
|
||||
# pull last assistant text
|
||||
last = ""
|
||||
for line in out_f.read_text(errors="replace").splitlines():
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if ev.get("type") == "turn_end" and ev.get("message", {}).get("role") == "assistant":
|
||||
for c in ev["message"].get("content", []):
|
||||
if c.get("type") == "text" and c.get("text", "").strip():
|
||||
last = c["text"]
|
||||
(run_dir / f"final_answer_turn{n}.md").write_text(last or "(none)")
|
||||
return p.returncode
|
||||
|
||||
|
||||
def main():
|
||||
iteration, tag = sys.argv[1], sys.argv[2]
|
||||
for t in TASKS:
|
||||
rd = ROOT / iteration / t["name"] / "auto_skill" / "run-1"
|
||||
rd.mkdir(parents=True, exist_ok=True)
|
||||
sid = f"mt-{tag}-{t['name']}"
|
||||
t0 = time.time()
|
||||
r1 = turn(rd, sid, t["turn1"], 1)
|
||||
r2 = turn(rd, sid, t["turn2"], 2)
|
||||
print(f"{t['name']}: turn1 exit={r1} turn2 exit={r2} dur={round(time.time()-t0,1)}s", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user