#!/usr/bin/env python3
"""Multi-turn evals: two-round conversations where turn 2 corrects turn 1.

Verifies the agent AMENDS the previous artifacts (edit workflow / add field)
instead of ignoring the correction or rebuilding blindly.

Usage: python3 run_multiturn.py <iteration> <tag>
"""
import json, os, subprocess, sys, time
from pathlib import Path

HARNESS = Path(__file__).parent
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
MODEL = os.environ.get("EVAL_MODEL", "opencode-go/deepseek-v4.1-flash")
RUN_TIMEOUT = 480
PI_HOME = os.environ.get("EVAL_PI_HOME_SKILLED", "/root/eval-rerun/pi-home-skilled")  # trigger mode: skill auto-discovered

env_all = json.loads((HARNESS / "sandbox-env.json").read_text())

TASKS = [
    {
        "name": "mt-1-notify-correction",
        "turn1": "帮我建个自动化:order_logs 表有新记录的时候通知我。名字你来定,配好激活。",
        "turn2": "不对,改一下:别用邮件,要站内通知;而且只在「结果」字段变成「失败」的时候才通知,不是有新记录就通知。",
        "turn1_hint": None,
    },
    {
        "name": "mt-2-incremental-schema",
        "turn1": "帮我建张表存客户反馈,加个单行文本「内容」字段,随便插一条示例。",
        "turn2": "再补一个单选字段「渠道」,选项:微信/电话/邮件。然后把刚才那条示例记录的渠道标成「微信」。",
        "turn1_hint": None,
    },
]

WRAP = """要求:
1. 逐步用 CLI 完成实际操作(不是只给命令)。
2. 每轮结束后把当前轮的答复写入 final_answer_turn{N}.md(做了什么、关键命令、结果)。
3. 用户的后续修正优先于之前的做法;能改就改现有的,不要无视."""


def turn(run_dir: Path, sid: str, text: str, n: int):
    prompt = f"{text}\n\n{WRAP.format(N=n)}"
    cmd = ["timeout", str(RUN_TIMEOUT), "pi", "-p", "--mode", "json",
           "--session-id", sid, "--model", MODEL, prompt]
    out_f = run_dir / f"turn{n}.jsonl"
    env = {
        **os.environ, "HOME": PI_HOME,
        "PATH": f"/root/eval-rerun/bin-new/bin:{os.environ['PATH']}",
        "INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
        "INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
        "INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"],
    }
    with open(out_f, "w") as out:
        p = subprocess.run(cmd, stdout=out, stderr=subprocess.DEVNULL, cwd=run_dir, env=env)
    # pull last assistant text
    last = ""
    for line in out_f.read_text(errors="replace").splitlines():
        try:
            ev = json.loads(line)
        except json.JSONDecodeError:
            continue
        if ev.get("type") == "turn_end" and ev.get("message", {}).get("role") == "assistant":
            for c in ev["message"].get("content", []):
                if c.get("type") == "text" and c.get("text", "").strip():
                    last = c["text"]
    (run_dir / f"final_answer_turn{n}.md").write_text(last or "(none)")
    return p.returncode


def main():
    iteration, tag = sys.argv[1], sys.argv[2]
    for t in TASKS:
        rd = ROOT / iteration / t["name"] / "auto_skill" / "run-1"
        rd.mkdir(parents=True, exist_ok=True)
        sid = f"mt-{tag}-{t['name']}"
        t0 = time.time()
        r1 = turn(rd, sid, t["turn1"], 1)
        r2 = turn(rd, sid, t["turn2"], 2)
        print(f"{t['name']}: turn1 exit={r1} turn2 exit={r2} dur={round(time.time()-t0,1)}s", flush=True)


if __name__ == "__main__":
    main()