yeuimu
c9694511a1
- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖) - SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引 (snapshot 无恢复 API,用 deactivate+节点改回) - hint 补 429 限流与超时模式 - evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/ EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分 - 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用 - 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次; 多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话) - 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
84 lines
3.5 KiB
Python
84 lines
3.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Multi-turn evals: two-round conversations where turn 2 corrects turn 1.
|
|
|
|
Verifies the agent AMENDS the previous artifacts (edit workflow / add field)
|
|
instead of ignoring the correction or rebuilding blindly.
|
|
|
|
Usage: python3 run_multiturn.py <iteration> <tag>
|
|
"""
|
|
import json, os, subprocess, sys, time
|
|
from pathlib import Path
|
|
|
|
HARNESS = Path(__file__).parent
|
|
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
|
|
MODEL = os.environ.get("EVAL_MODEL", "opencode-go/deepseek-v4.1-flash")
|
|
RUN_TIMEOUT = 480
|
|
PI_HOME = os.environ.get("EVAL_PI_HOME_SKILLED", "/root/eval-rerun/pi-home-skilled") # trigger mode: skill auto-discovered
|
|
|
|
env_all = json.loads((HARNESS / "sandbox-env.json").read_text())
|
|
|
|
TASKS = [
|
|
{
|
|
"name": "mt-1-notify-correction",
|
|
"turn1": "帮我建个自动化:order_logs 表有新记录的时候通知我。名字你来定,配好激活。",
|
|
"turn2": "不对,改一下:别用邮件,要站内通知;而且只在「结果」字段变成「失败」的时候才通知,不是有新记录就通知。",
|
|
"turn1_hint": None,
|
|
},
|
|
{
|
|
"name": "mt-2-incremental-schema",
|
|
"turn1": "帮我建张表存客户反馈,加个单行文本「内容」字段,随便插一条示例。",
|
|
"turn2": "再补一个单选字段「渠道」,选项:微信/电话/邮件。然后把刚才那条示例记录的渠道标成「微信」。",
|
|
"turn1_hint": None,
|
|
},
|
|
]
|
|
|
|
WRAP = """要求:
|
|
1. 逐步用 CLI 完成实际操作(不是只给命令)。
|
|
2. 每轮结束后把当前轮的答复写入 final_answer_turn{N}.md(做了什么、关键命令、结果)。
|
|
3. 用户的后续修正优先于之前的做法;能改就改现有的,不要无视."""
|
|
|
|
|
|
def turn(run_dir: Path, sid: str, text: str, n: int):
|
|
prompt = f"{text}\n\n{WRAP.format(N=n)}"
|
|
cmd = ["timeout", str(RUN_TIMEOUT), "pi", "-p", "--mode", "json",
|
|
"--session-id", sid, "--model", MODEL, prompt]
|
|
out_f = run_dir / f"turn{n}.jsonl"
|
|
env = {
|
|
**os.environ, "HOME": PI_HOME,
|
|
"PATH": f"/root/eval-rerun/bin-new/bin:{os.environ['PATH']}",
|
|
"INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
|
|
"INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
|
|
"INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"],
|
|
}
|
|
with open(out_f, "w") as out:
|
|
p = subprocess.run(cmd, stdout=out, stderr=subprocess.DEVNULL, cwd=run_dir, env=env)
|
|
# pull last assistant text
|
|
last = ""
|
|
for line in out_f.read_text(errors="replace").splitlines():
|
|
try:
|
|
ev = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if ev.get("type") == "turn_end" and ev.get("message", {}).get("role") == "assistant":
|
|
for c in ev["message"].get("content", []):
|
|
if c.get("type") == "text" and c.get("text", "").strip():
|
|
last = c["text"]
|
|
(run_dir / f"final_answer_turn{n}.md").write_text(last or "(none)")
|
|
return p.returncode
|
|
|
|
|
|
def main():
|
|
iteration, tag = sys.argv[1], sys.argv[2]
|
|
for t in TASKS:
|
|
rd = ROOT / iteration / t["name"] / "auto_skill" / "run-1"
|
|
rd.mkdir(parents=True, exist_ok=True)
|
|
sid = f"mt-{tag}-{t['name']}"
|
|
t0 = time.time()
|
|
r1 = turn(rd, sid, t["turn1"], 1)
|
|
r2 = turn(rd, sid, t["turn2"], 2)
|
|
print(f"{t['name']}: turn1 exit={r1} turn2 exit={r2} dur={round(time.time()-t0,1)}s", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|