#!/usr/bin/env python3
"""Grade plugin-skill eval runs (ported from plugin-workspace, per-config sandboxes)."""
import json, os, re, subprocess, sys
from pathlib import Path

HARNESS = Path(__file__).parent  # evals/ itself
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable")
env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'plugin-sandbox-env.json'))).read_text())

FORBIDDEN_IMPORT = re.compile(r"^\s*import\s+(?!type)[^\n]*from\s+['\"]@teable/(sdk|ui-lib|openapi|core|icons)", re.M)
REACT_IMPORT = re.compile(r"^\s*import\s+(?!type)[^\n]*from\s+['\"]react['\"]", re.M)


def cli_json(*args):
    e = {**os.environ,
         "INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
         "INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
         "INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"]}
    p = subprocess.run([CLI, *args, "--json"], capture_output=True, text=True, env=e, timeout=60)
    try:
        return json.loads(p.stdout)
    except json.JSONDecodeError:
        return None


def transcript(run_dir: Path):
    cmds = []
    for line in (run_dir / "pi.jsonl").read_text(errors="replace").splitlines():
        try:
            ev = json.loads(line)
        except json.JSONDecodeError:
            continue
        if ev.get("type") == "tool_execution_start" and ev.get("toolName") == "bash":
            cmds.append(ev.get("args", {}).get("command", ""))
    return cmds


def answer(run_dir: Path) -> str:
    fa, af = run_dir / "final_answer.md", run_dir / "assistant_final.md"
    return fa.read_text(errors="replace") if fa.exists() else af.read_text(errors="replace")


def plugin_sources(run_dir: Path):
    files = sorted((run_dir / "plugin").rglob("*.[tj]s*")) or sorted(run_dir.rglob("plugin.json"))
    return "\n".join(p.read_text(errors="replace") for p in files), files


def result(text, passed, evidence):
    return {"text": text, "passed": bool(passed), "evidence": str(evidence)[:400]}


def grade_eval(ev_name, run_dir: Path, cfg_short: str):
    cmds = transcript(run_dir)
    ans = answer(run_dir)
    out = []

    if ev_name in ("contextmenu-plugin", "panel-widget"):
        src, files = plugin_sources(run_dir)
        want_pos = "contextMenu" if ev_name == "contextmenu-plugin" else "panel"
        want_api = "addContextMenuAction" if ev_name == "contextmenu-plugin" else "addPanel"
        pj = run_dir / "plugin" / "plugin.json"
        manifest_ok = False
        if pj.exists():
            try:
                m = json.loads(pj.read_text())
                manifest_ok = bool(re.fullmatch(r"[a-z0-9-]+", m.get("id", ""))) and want_pos in (m.get("positions") or [])
            except json.JSONDecodeError:
                pass
        out.append(result(f"plugin.json 存在且 manifest 合法(id 合规、positions 含 {want_pos})", manifest_ok,
                          pj.read_text(errors="replace")[:150] if pj.exists() else "plugin/plugin.json 未找到"))
        out.append(result("代码从 window.__TEABLE_HOST__ 获取宿主能力", "__TEABLE_HOST__" in src,
                          "host global: %s" % ("有" if "__TEABLE_HOST__" in src else "无")))
        bad = FORBIDDEN_IMPORT.findall(src)
        out.append(result("@teable/* 无运行时值导入(仅允许 import type)", not bad, "违规: %r" % bad if bad else "ok"))
        uses_react = bool(REACT_IMPORT.search(src))
        wp = run_dir / "plugin" / "webpack.config.js"
        wp_txt = wp.read_text(errors="replace") if wp.exists() else ""
        externals_ok = "__TEABLE_HOST__" in wp_txt and "externals" in wp_txt
        out.append(result("react 导入经 externals 映射(或无 react 裸导入)", externals_ok if uses_react else True,
                          "webpack externals: %s" % ("映射" if externals_ok else ("未映射!" if uses_react else "不需要"))))
        out.append(result(f"definePlugin 注册且调用 ctx.{want_api}", "definePlugin" in src and want_api in src,
                          "definePlugin=%s %s=%s" % ("definePlugin" in src, want_api, want_api in src)))
        if ev_name == "contextmenu-plugin":
            edits = re.search(r"updateRecords|record.*update|OpenAPI", src, re.I)
            out.append(result("实现了批量更新记录逻辑", bool(edits), edits.group(0) if edits else "未发现"))
        else:
            sel = re.search(r"useSelection|selection|getSelectionRecords", src, re.I)
            out.append(result("通过宿主 SDK 获取选中记录", bool(sel), sel.group(0) if sel else "未发现"))

    elif ev_name == "deploy-chain":
        plugin_id = f"eval-deploy-{cfg_short}"
        up = any("upload-bundle" in c for c in cmds)
        out.append(result("调用了 upload-bundle 上传", up, [c for c in cmds if "upload-bundle" in c][:1]))
        enabled = any("plugin-config" in c for c in cmds)
        out.append(result("启用了 plugin-config", enabled, [c for c in cmds if "plugin-config" in c][:1]))
        installed = any(re.search(r"context-menu.*install|plugin-context-menu", c) for c in cmds)
        out.append(result("安装到表级 context-menu 位置", installed, [c for c in cmds if "context-menu" in c][:1]))
        cleaned = any(("DELETE" in c.upper() and "/api/plugin/" in c) or re.search(r"plugin uninstall|plugin-install", c) for c in cmds) \
            or (cli_json("plugin", "list") or [] and not any(p.get("id") == plugin_id for p in (cli_json("plugin", "list") or [])) and any("install" in c for c in cmds))
        out.append(result("执行了清理(卸载安装/删除插件)", cleaned, [c for c in cmds if "uninstall" in c or "DELETE" in c.upper()][:2]))
        lst = json.dumps(cli_json("plugin", "list") or [], ensure_ascii=False)
        out.append(result("实例上无插件残留(plugin list 不含该 id)", plugin_id not in lst,
                          "残留!" if plugin_id in lst else "干净"))
        verify = re.search(r"已确认|确认(安装|清理|删除)|installed|verified|清理(完成|干净)", ans)
        out.append(result("答复确认了安装与清理结果", bool(verify), ans[:200]))

    elif ev_name == "proxy-rule":
        out.append(result("方案使用同源代理 /api/plugin-proxy/<service>/...", "/api/plugin-proxy/" in ans, ans[:250]))
        out.append(result("指出需注册 PLUGIN_API_SERVICES 并重启 teable-backend", "PLUGIN_API_SERVICES" in ans,
                          "提及" if "PLUGIN_API_SERVICES" in ans else "未提及"))
        lan_fetch = re.search(r"fetch\(\s*['\"]http://192\.168", ans)
        out.append(result("不建议在 bundle 里直接请求局域网 IP", lan_fetch is None, lan_fetch.group(0) if lan_fetch else "ok"))

    elif ev_name == "pitfall-diagnosis":
        cause = re.search(r"(@teable/sdk|值导入|运行时导入|externals|undefined)|import", ans) and re.search(r"__TEABLE_HOST__|宿主全局|host global", ans)
        out.append(result("定位为 @teable/* 运行时导入被 externals 映射为 undefined", bool(cause), ans[:250]))
        out.append(result("给出修复:改从 window.__TEABLE_HOST__ 读取(或仅 import type)", "__TEABLE_HOST__" in ans, ans[:200]))

    elif ev_name == "native-form-routing":
        nf = re.search(r"NativeForm|useNativeForm|NativeFormScope", ans)
        out.append(result("指出原生表单注入能力(NativeForm/NativeFormScope/useNativeForm)", bool(nf),
                          nf.group(0) if nf else ans[:200]))
        # 咨询类任务应给出可实现方案(两档之一 + 关键代码/props)
        plan = bool(nf) and bool(re.search(r"visibleFieldIds|slots|NativeFormField|formId|interceptSubmit", ans))
        out.append(result("给出了具体实现要点(props/多步结构)", plan, ans[:250]))

    return out


def main():
    iteration_dir = ROOT / sys.argv[1]
    for eval_dir in sorted(iteration_dir.glob("eval-*")):
        ev_name = eval_dir.name.split("-", 2)[2]
        for config_dir in sorted(p for p in eval_dir.iterdir() if p.is_dir()):
            run_dir = config_dir / "run-1"
            if not (run_dir / "pi.jsonl").exists():
                continue
            cfg_short = "W" if config_dir.name == "with_skill" else "O"
            try:
                expectations = grade_eval(ev_name, run_dir, cfg_short)
            except Exception as e:
                expectations = [result("判分异常", False, repr(e))]
            (run_dir / "grading.json").write_text(json.dumps({"expectations": expectations}, indent=2, ensure_ascii=False))
            npass = sum(1 for x in expectations if x["passed"])
            print(f"{eval_dir.name:34} {config_dir.name:12} {npass}/{len(expectations)}", flush=True)


if __name__ == "__main__":
    main()