#!/usr/bin/env python3
"""Launch inkreach-teable-plugin skill evals (integrity-clean port).

Usage: python3 run_plugin_evals.py <iteration> <tag> <config-spec> [eval-ids]
  config-spec: with_skill=/root/eval-rerun/skill-plugin-new:/root/eval-rerun/bin-new/bin
"""
import json, os, re, shutil, subprocess, sys, time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

HARNESS = Path(__file__).parent  # evals/ itself
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
MODEL = os.environ.get("EVAL_MODEL", "opencode-go/deepseek-v4.1-flash")
RUN_TIMEOUT = 900  # plugin evals build bundles — need more headroom
PI_HOME = os.environ.get("EVAL_PI_HOME", "/root/eval-rerun/pi-home")
FIXTURE = "/root/pi-skill-lab/plugin-workspace/fixture-deploy-W"

env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'plugin-sandbox-env.json'))).read_text())

PROMPT_TMPL = """你在为一套已部署的 inkreach-teable 实例(Teable fork)开发/部署动态 React 插件。

环境(已配置好,直接使用):
- 后端 http://localhost:3003,前端 http://localhost:3002。PAT 已通过环境变量 INKREACH_TEABLE_ENDPOINT / INKREACH_TEABLE_TOKEN 注入(`inkreach-teable` CLI 可直接用)。
- 评估沙箱基地 ID:{BASE}。凡涉及实例的读写一律只在这个基地内,严禁触碰其他基地;严禁向实例的插件注册表上传与任务无关的演示插件。
- 工作目录就是当前目录,所有产出写在这里。

{SKILL_LINE}

任务:{TASK}

要求:
1. 按任务性质完成:咨询类问题(能不能/怎么做)直接给出方案与关键代码即可,不必搭建完整工程;开发/部署类任务才需要实际构建与部署。
2. 完成后把最终答复写入 final_answer.md:结论、关键实现/命令、结果、需要用户知道的事项。"""


def extract_final_text(p: Path) -> str:
    texts = []
    for line in p.read_text(errors="replace").splitlines():
        try:
            ev = json.loads(line)
        except json.JSONDecodeError:
            continue
        if ev.get("type") == "turn_end" and ev.get("message", {}).get("role") == "assistant":
            for c in ev["message"].get("content", []):
                if c.get("type") == "text" and c.get("text", "").strip():
                    texts.append(c["text"])
    return texts[-1] if texts else ""


def run_one(task, config, skill_path, bin_dir, cfg_env, run_dir: Path):
    run_dir.mkdir(parents=True, exist_ok=True)
    skill_line = (
        f"- 先完整阅读技能说明 {skill_path}/SKILL.md 并按其模块地图(按需读 dynamic-plugins/ 下文档)操作。"
        if skill_path else ""
    )
    prompt = PROMPT_TMPL.format(BASE=cfg_env["INKREACH_TEABLE_BASE_ID"], SKILL_LINE=skill_line, TASK=task)
    (run_dir / "prompt.txt").write_text(prompt)
    full_env = {
        **os.environ, "HOME": PI_HOME, "PATH": f"{bin_dir}:{os.environ['PATH']}",
        **{k: env_all[k] for k in ("INKREACH_TEABLE_ENDPOINT", "INKREACH_TEABLE_TOKEN")},
        **cfg_env,
    }
    cmd = ["timeout", str(RUN_TIMEOUT), "pi", "-p", "--mode", "json", "--model", MODEL, prompt]
    t0 = time.time()
    with open(run_dir / "pi.jsonl", "w") as out:
        proc = subprocess.run(cmd, stdout=out, stderr=subprocess.DEVNULL, cwd=run_dir, env=full_env)
    (run_dir / "timing.json").write_text(json.dumps(
        {"duration_s": round(time.time() - t0, 1), "exit_code": proc.returncode, "config": config}))
    (run_dir / "assistant_final.md").write_text(extract_final_text(run_dir / "pi.jsonl") or "(none)")
    return config, run_dir.name, proc.returncode


def main():
    iteration, tag = sys.argv[1], sys.argv[2]
    iteration_dir = ROOT / iteration
    configs = {}
    for part in sys.argv[3].split(","):
        name, _, rest = part.partition("=")
        sp, _, bd = rest.partition(":")
        configs[name] = (sp, bd)
    evals = json.loads(Path(os.environ.get('EVAL_EVALS', str(HARNESS / 'plugin-evals.json'))).read_text())["evals"]
    if len(sys.argv) > 4 and sys.argv[4]:
        only = {int(x) for x in sys.argv[4].split(",")}
        evals = [e for e in evals if e["id"] in only]

    jobs = []
    for ev in evals:
        name = f"eval-{ev['id']}-{ev['name']}"
        for config, (skill_path, bin_dir) in configs.items():
            cfg = env_all[config]
            cfg_short = "W" if config == "with_skill" else "O"
            task = ev["prompt"].replace("{TBL}", cfg["TBL"])
            rd = iteration_dir / name / config / "run-1"
            rd.parent.mkdir(parents=True, exist_ok=True)
            # eval-3 needs the ready-plugin fixture bundle in cwd
            if ev["id"] == 3:
                rp = rd / "ready-plugin"
                if rp.exists():
                    shutil.rmtree(rp)
                shutil.copytree(FIXTURE, rp)
            (rd.parent / "eval_metadata.json").write_text(json.dumps(
                {"eval_id": ev["id"], "prompt": task, "assertions": []}, ensure_ascii=False))
            cfg_env = {"INKREACH_TEABLE_BASE_ID": cfg["INKREACH_TEABLE_BASE_ID"]}
            jobs.append((task, config, skill_path, bin_dir, cfg_env, rd))

    print(f"{len(jobs)} runs -> {iteration_dir}", flush=True)
    with ThreadPoolExecutor(max_workers=3) as ex:  # builds are heavy — 3 workers
        for f in [ex.submit(run_one, *j) for j in jobs]:
            print(f.result(), flush=True)


if __name__ == "__main__":
    main()