#!/usr/bin/env python3
"""Grade iteration-4 A/B rerun evals.

Same checks as the original grade_evals.py, adapted to per-config sandboxes:
- each config (with_skill/old_skill) has its own base + order_logs table + ORD record
- CLI = absolute path to the NEW CLI (reads only; version-agnostic)
Usage: python3 grade_evals4.py <iteration-name> <tag>
"""
import json
import os
import re
import subprocess
import sys
from pathlib import Path

HARNESS = Path(__file__).parent  # evals/ itself
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable")
env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'sandbox-env.json'))).read_text())


def cli_json(cfg, *args):
    e = {**os.environ,
         "INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
         "INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
         "INKREACH_TEABLE_BASE_ID": cfg["INKREACH_TEABLE_BASE_ID"]}
    p = subprocess.run([CLI, *args, "--json"], capture_output=True, text=True, env=e, timeout=60)
    try:
        return json.loads(p.stdout)
    except json.JSONDecodeError:
        return None


def transcript(run_dir: Path):
    cmds, reads = [], []
    for line in (run_dir / "pi.jsonl").read_text(errors="replace").splitlines():
        line = line.strip()
        if not line:
            continue
        try:
            ev = json.loads(line)
        except json.JSONDecodeError:
            continue
        if ev.get("type") != "tool_execution_start":
            continue
        if ev.get("toolName") == "bash":
            cmds.append(ev.get("args", {}).get("command", ""))
        elif ev.get("toolName") == "read":
            reads.append(ev.get("args", {}).get("path", ""))
    return cmds, reads


def answer(run_dir: Path) -> str:
    fa = run_dir / "final_answer.md"
    af = run_dir / "assistant_final.md"
    return (fa.read_text(errors="replace") if fa.exists() else af.read_text(errors="replace"))


def used_official_teable(cmds):
    for c in cmds:
        # probing for existence (command -v / which) is prudent, not usage
        probe = re.search(r"(command -v|which)\s+teable", c)
        if probe:
            continue
        if re.search(r"(^|[^a-z-])teable\s", c) or "npx teable" in c or re.search(r"npm\s+(i|install)\s+teable\b", c):
            return c
    return None


def find_wf(cfg, name_part):
    wfs = cli_json(cfg, "workflow", "list") or []
    if not isinstance(wfs, list):
        return None  # error payload ({"error":...}) — not a workflow list
    for w in wfs:
        if isinstance(w, dict) and name_part in w.get("name", ""):
            return w
    return None


def wf_nodes(cfg, wf_id):
    wf = cli_json(cfg, "workflow", "get", wf_id) or {}
    return wf, [n for n in wf.get("nodes", []) if n.get("type") == "trigger"], [n for n in wf.get("nodes", []) if n.get("type") == "action"]


def result(text, passed, evidence):
    return {"text": text, "passed": bool(passed), "evidence": str(evidence)[:400]}


def grade_eval(cfg, ev_name, tag, run_dir, cfg_short="N"):
    tbl = cfg["TBL5_ID"]
    ord_id = cfg["ORD"]
    cmds, reads = transcript(run_dir)
    ans = answer(run_dir)
    all_cmds = "\n".join(cmds)
    out = []

    if ev_name == "button-click-automation":
        # Locate the workflow via the button-field binding (options.workflow) —
        # the prompt only fixes the FIELD name, agents may name the workflow
        # anything (e.g. 催单自动化A4W), so searching by workflow name misfires.
        fields = cli_json(cfg, "field", "list", tbl) or []
        btn = next((f for f in fields if f.get("name") == f"催单按钮{tag}"), None)
        wf = None
        if btn:
            wf_id = ((btn.get("options") or {}).get("workflow") or {}).get("id")
            if wf_id:
                wf = cli_json(cfg, "workflow", "get", wf_id)
        if not wf:
            wf = find_wf(cfg, f"催单按钮{tag}")
        if not wf:
            out.append(result("按钮字段绑定的工作流存在 (催单按钮%s)" % tag, False,
                              "field list 无该按钮字段或其 options.workflow 无效"))
            return out
        wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
        out.append(result("工作流已激活 (isActive)", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
        fields = cli_json(cfg, "field", "list", tbl) or []
        btn = next((f for f in fields if f.get("name") == f"催单按钮{tag}"), None)
        if not btn:
            out.append(result("创建按钮字段 催单按钮%s" % tag, False, "field list 未找到该字段"))
        else:
            wf_opt = (btn.get("options") or {}).get("workflow")
            ok = isinstance(wf_opt, dict) and wf_opt.get("id") == wf["id"]
            out.append(result("按钮字段 options.workflow 为对象且指向该工作流", ok,
                              "options.workflow=%r" % wf_opt))
        out.append(result("触发器为 buttonClick", any(t.get("category") == "buttonClick" for t in triggers),
                          "trigger categories=%s" % [t.get("category") for t in triggers]))
        script = next((a for a in actions if a.get("category") == "script"), None)
        out.append(result("script action 读取 triggerInput", script and "triggerInput" in (script.get("config") or {}).get("code", ""),
                          "code=%r" % ((script or {}).get("config") or {}).get("code", "")[:120]))
        fired = any("fire button-click" in c for c in cmds)
        runs_any = cli_json(cfg, "runs", "list", wf["id"]) or []
        verified = any(re.search(r"runs (list|get)", c) for c in cmds) or len(runs_any) > 0
        out.append(result("用 fire button-click 实测", fired, [c for c in cmds if "fire" in c][:2]))
        out.append(result("执行后查看 runs 结果", verified, [c for c in cmds if "runs" in c][:2]))
        runs = cli_json(cfg, "runs", "list", wf["id"]) or []
        ok_run = any(r.get("status") in ("succeeded", "success") for r in runs)
        out.append(result("存在 succeeded 的执行记录", ok_run, "statuses=%s" % [r.get("status") for r in runs][:5]))
        bad = used_official_teable(cmds)
        out.append(result("全程使用 inkreach-teable CLI(未用官方 teable)", bad is None, bad or "ok"))

    elif ev_name == "scheduled-digest":
        wf = find_wf(cfg, f"每日统计{tag}")
        if not wf:
            out.append(result("创建名为 每日统计%s 的工作流" % tag, False, "workflow list 中未找到"))
            return out
        wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
        out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
        trig = triggers[0] if triggers else {}
        timing = (trig.get("config") or {}).get("timing") or {}
        ignored = [k for k in ("triggerTime", "weekdays", "daysOfMonth", "starting", "ending", "timezone", "tz") if k in timing]
        out.append(result("scheduledTime 触发器", trig.get("category") == "scheduledTime", "category=%s" % trig.get("category")))
        out.append(result("timing 只含生效字段(type+interval,无被忽略字段)", trig.get("category") != "scheduledTime" or (not ignored and "type" in timing and "interval" in timing),
                          "timing=%r ignored=%s" % (timing, ignored)))
        has_stats = any(a.get("category") == "script" and re.search(r"record|count|统计|length", (a.get("config") or {}).get("code", "")) for a in actions) or any(
            a.get("category") in ("getRecords",) for a in actions)
        out.append(result("有统计逻辑的 action(script 读记录数或 getRecords)", has_stats,
                          "action categories=%s" % [a.get("category") for a in actions]))

    elif ev_name == "record-condition-email":
        wf = find_wf(cfg, f"失败告警{tag}")
        if not wf:
            out.append(result("创建名为 失败告警%s 的工作流" % tag, False, "workflow list 中未找到"))
            return out
        wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
        out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
        trig = triggers[0] if triggers else {}
        out.append(result("触发器 category 为 recordCondition", trig.get("category") == "recordCondition",
                          "category=%s" % trig.get("category")))
        c = trig.get("config") or {}
        cond_ok = "conditions" in c and "filter" not in c
        out.append(result("条件键名为 conditions(无 filter)", cond_ok, "config keys=%s" % list(c.keys())))
        if isinstance(c.get("conditions"), dict):
            out.append(result("conditions 形如 {conjunction, filterSet}", "filterSet" in c["conditions"],
                              "conditions keys=%s" % list(c["conditions"].keys())))
        bad_key = "recordMatchesConditions" in all_cmds
        out.append(result("未使用官方键名 recordMatchesConditions", not bad_key, "transcript" if bad_key else "ok"))
        email = next((a for a in actions if a.get("category") in ("sendEmail", "email")), None)
        ok_email = email and "ops@xingchen.tech" in json.dumps(email.get("config") or {}, ensure_ascii=False)
        out.append(result("sendEmail action 收件人 ops@xingchen.tech", ok_email,
                          "email action config=%s" % json.dumps((email or {}).get("config") or {}, ensure_ascii=False)[:200]))

    elif ev_name == "debug-failed-run":
        out.append(result("查询了工作流状态 (workflow get/list)", any(re.search(r"workflow (get|list)", c) for c in cmds),
                          [c for c in cmds if "workflow" in c][:2]))
        out.append(result("查询失败执行 (runs list --status failed 或过滤)", any(re.search(r"runs list", c) for c in cmds),
                          [c for c in cmds if "runs" in c][:3]))
        detail_in_ans = bool(re.search(r"boom|nodeResults|错误详情|执行失败原因|throw new Error", ans))
        out.append(result("查看了具体执行详情 (runs get 或答复引用运行错误详情)",
                          any(re.search(r"runs get", c) for c in cmds) or detail_in_ans,
                          [c for c in cmds if "runs get" in c][:2] or ans[:120]))
        diagnosed = re.search(r"boom|timeout|超时|抛错|throw", ans, re.I)
        out.append(result("最终答复指出脚本抛错原因", bool(diagnosed), ans[:200]))
        reactivated = [c for c in cmds if re.search(r"workflow activate", c)]
        out.append(result("未盲目重新激活(诊断前不改)", not reactivated, reactivated or "ok"))

    elif ev_name == "select-constraint-write":
        inspected = any(re.search(r"field (list|get|visible-choices)", c) for c in cmds)
        out.append(result("写入前检查了字段约束 (field list --with-options / field get)", inspected,
                          [c for c in cmds if "field" in c][:2]))
        fields = cli_json(cfg, "field", "list", tbl) or []
        f = next((x for x in fields if x.get("name") == "结果"), {})
        choices = [c.get("name") if isinstance(c, dict) else c for c in ((f.get("options") or {}).get("choices") or [])]
        added = "已完成" in choices
        recs = (cli_json(cfg, "record", "get", "--table-id", tbl) or {}).get("records") or []
        rec = next((r for r in recs if (r.get("fields") or {}).get("订单号") == ord_id), {})
        val = (rec.get("fields") or {}).get("结果")
        claims_success = bool(re.search(r"(✅|已.{0,3}(改|更新|设置|写入).{0,20}已完成|成功.{0,10}改为)", ans))
        if added and val == "已完成":
            out.append(result("合法路径:先加选项再写入,记录值=已完成", True, "choices=%s val=%s" % (choices, val)))
        elif claims_success and val != "已完成" and not added:
            out.append(result("未发生静默失败(声称成功但值未变)", False,
                              "答复声称已改为「已完成」,但实际 choices=%s 记录值=%r —— 教科书式 mutationApplied=false 静默失败" % (choices, val)))
        elif val in (None, "处理中") and re.search(r"不在|无法|无效|不是.{0,6}选项|未写入|没有写入|未添加|无法直接|不能直接", ans):
            out.append(result("识别出「已完成」不是有效选项并向用户说明(未静默失败)", True, ans[:250]))
        else:
            out.append(result("写入了有效选项值或正确说明", val == "已完成",
                              "choices=%s val=%r claims_success=%s ans=%s" % (choices, val, claims_success, ans[:150])))
        listed = all(x in ans for x in ("成功", "失败", "处理中"))
        out.append(result("答复列出了全部可选值(成功/失败/处理中)", listed, ans[:200]))

    elif ev_name == "official-cli-near-miss":
        bad = used_official_teable(cmds)
        out.append(result("未安装/调用官方 teable CLI 打实例", bad is None, bad or "ok"))
        out.append(result("使用了 inkreach-teable CLI", any("inkreach-teable" in c for c in cmds),
                          [c for c in cmds if "inkreach-teable" in c][:2]))
        out.append(result("答复列出了 order_logs 表", "order_logs" in ans, ans[:200]))

    elif ev_name == "webhook-receiver":
        wf = find_wf(cfg, f"回单通知{tag}")
        if not wf:
            out.append(result("创建名为 回单通知%s 的工作流" % tag, False, "workflow list 中未找到"))
            return out
        wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
        out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
        trig = triggers[0] if triggers else {}
        out.append(result("触发器为 webhook", trig.get("category") == "webhook", "category=%s" % trig.get("category")))
        base = cfg["INKREACH_TEABLE_BASE_ID"]
        correct = f"/api/webhook/{base}/{wf['id']}"
        out.append(result("答复给出正确 URL 形状 /api/webhook/{baseId}/{workflowId}(无 /base/ 段)", correct in ans,
                          "期望含 %s;答复=%s" % (correct, ans[:180])))
        tested = any(re.search(r"fire webhook", c) for c in cmds) or any("api/webhook" in c and re.search(r"curl|POST", c) for c in cmds)
        out.append(result("实际测试了 webhook(fire webhook 或 curl POST)", tested,
                          [c for c in cmds if "webhook" in c][:3]))

    elif ev_name == "weekday-schedule-trap":
        wf = find_wf(cfg, f"工作日盘点{tag}")
        if not wf:
            out.append(result("创建名为 工作日盘点%s 的工作流" % tag, False, "workflow list 中未找到"))
            return out
        wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
        out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
        trig = triggers[0] if triggers else {}
        timing = (trig.get("config") or {}).get("timing") or {}
        ignored = [k for k in ("triggerTime", "weekdays", "daysOfMonth", "starting", "ending", "timezone", "tz") if k in timing]
        out.append(result("scheduledTime 触发器且 timing 只含生效字段", trig.get("category") == "scheduledTime" and not ignored and "type" in timing,
                          "category=%s timing=%r ignored=%s" % (trig.get("category"), timing, ignored)))
        warns = re.search(r"(忽略|不支持|仅|只).{0,30}(interval|间隔|type)|weekday|工作日.{0,10}(不|无法|忽略)|backend ignores", ans, re.I)
        out.append(result("答复如实告知『具体时刻/星期』不被后端支持", bool(warns), ans[:250]))
        claims_exact = re.search(r"(已.{0,6}(配好|设置).{0,40}(工作日|下午\s*5|17:00))|每个工作日下午 ?5 ?点.{0,10}(会|将)跑", ans)
        out.append(result("未虚假承诺『工作日下午5点会照常执行』", not claims_exact, ans[:200]))

    elif ev_name == "secret-env-usage":
        wf = find_wf(cfg, f"汇率查询{tag}")
        if not wf:
            out.append(result("创建名为 汇率查询%s 的工作流" % tag, False, "workflow list 中未找到"))
            return out
        wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
        out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
        secrets = cli_json(cfg, "secret", "list", wf["id"]) or []
        has_secret = bool(secrets)
        out.append(result("存了密钥(非明文入代码;secret set 或 secret list 可见)",
                          any(re.search(r"secret set", c) for c in cmds) or has_secret,
                          [c for c in cmds if "secret set" in c][:1] or f"secret list: {len(secrets)} 条"))
        script = next((a for a in actions if a.get("category") == "script"), None)
        code = (script or {}).get("config", {}).get("code", "")
        out.append(result("脚本经 process.env 读取密钥", bool(re.search(r"process\.env\.[A-Z_]+", code)), code[:150]))
        out.append(result("脚本/工作流配置中无明文 key(sk-test-abc123)", "sk-test-abc123" not in json.dumps(wf_full, ensure_ascii=False),
                          "明文检查通过" if "sk-test-abc123" not in json.dumps(wf_full, ensure_ascii=False) else "配置含明文!"))

    elif ev_name == "script-npm-deps":
        wf = find_wf(cfg, f"日期演示{tag}")
        if not wf:
            out.append(result("创建名为 日期演示%s 的工作流" % tag, False, "workflow list 中未找到"))
            return out
        wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
        out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
        script = next((a for a in actions if a.get("category") == "script"), None)
        code = (script or {}).get("config", {}).get("code", "") if script else ""
        out.append(result("脚本使用 dayjs", "dayjs" in code, code[:150]))
        out.append(result("依赖经 --dependencies 声明(配置含 dayjs)", "dayjs" in json.dumps((script or {}).get("config", {})),
                          json.dumps((script or {}).get("config", {}), ensure_ascii=False)[:200]))
        out.append(result("知晓后端 npm PATH 前提(答复或命令体现)", any(re.search(r"--dependencies", c) for c in cmds) or bool(re.search(r"PATH|npm|后端|service", ans)),
                          ans[:180]))

    elif ev_name == "lowperm-403-handling":
        ans_l = ans.lower()
        perm_ack = bool(re.search(r"权限|403|无权|forbidden|没.{0,4}(资格|权限)|令牌|token.{0,12}(无|不|失效)|授权", ans, re.I))
        fake_success = bool(re.search(r"(已(经)?(建|创建|完成)[了成]?.{0,10}(表|快递)|(表|快递).{0,8}(已|成功)(建|创建)|成功创建.{0,6}表)", ans))
        cli_tried = any("inkreach-teable" in c for c in cmds)
        out.append(result("尝试用 CLI 执行任务", cli_tried, [c for c in cmds if "inkreach-teable" in c][:1]))
        out.append(result("识别出权限不足并如实说明", perm_ack and not fake_success, ans[:220]))
        # 未陷入 403 循环(超过 8 次失败重试视为 flailing)
        f403 = sum(1 for c in cmds if re.search(r"inkreach-teable", c))
        out.append(result("未陷入无意义重试(命令次数有界)", f403 <= 12, f"cli calls={f403}"))

    elif ev_name == "deep-pagination":
        tbl13 = cfg.get("TBL13_ID")
        if not tbl13:
            out.append(result("fixture big_logs 存在", False, "TBL13_ID 未配置"))
            return out
        recs = (cli_json(cfg, "record", "get", "--table-id", tbl13, "--take", "1000") or {}).get("records") or []
        slow = [r for r in recs if str((r.get("fields") or {}).get("订单号", "")).startswith("SLOW-")]
        ids = [str((r.get("fields") or {}).get("订单号")) for r in slow]
        count_ok = bool(re.search(r"2\s*(条|个|records?|行)", ans))
        found_any = any(i in ans for i in ids)
        out.append(result("正确给出 SLOW- 记录总数=2", count_ok, f"实际: {ids}"))
        out.append(result("答复包含一条真实 SLOW- 记录的订单号", found_any, ans[:200]))

    elif ev_name == "vague-daily-notify":
        # 模糊需求:每天 + 订单进展 + 站内通知 → scheduledTime + 统计 + sendNotification + active
        wfs = cli_json(cfg, "workflow", "list") or []
        cands = [w for w in wfs if w.get("isActive")]
        hit = None
        for w in cands:
            wf_full, trs, acs = wf_nodes(cfg, w["id"])
            if (trs and trs[0].get("category") == "scheduledTime"
                    and any(a.get("category") == "sendNotification" for a in acs)):
                hit = wf_full
                break
        out.append(result("建了激活的每日定时工作流(scheduledTime+sendNotification)", hit is not None,
                          f"active scheduledTime+notify wf: {hit.get('name') if hit else None}"))
        if hit:
            _, trs, acs = wf_nodes(cfg, hit["id"])
            trig = trs[0] if trs else {}
            timing = (trig.get("config") or {}).get("timing") or {}
            out.append(result("timing 为生效字段(type+interval)", "type" in timing and "interval" in timing, f"timing={timing}"))
            has_stats = any(a.get("category") == "script" and re.search(r"record|count|统计|length", (a.get("config") or {}).get("code", "")) for a in acs)
            out.append(result("有订单统计逻辑", has_stats, [a.get("category") for a in acs]))

    elif ev_name == "field-options-update":
        fields = cli_json(cfg, "field", "list", tbl) or []
        f = next((x for x in fields if x.get("name") == "结果"), {})
        choices = [c.get("name") if isinstance(c, dict) else c for c in ((f.get("options") or {}).get("choices") or [])]
        out.append(result("「结果」字段 choices 新增「已取消」", "已取消" in choices, "choices=%s" % choices))
        out.append(result("使用 field update --options(走 convert API)", any(re.search(r"field update.*--options|field.*--options", c, re.S) for c in cmds),
                          [c for c in cmds if "field" in c and "update" in c][:1]))
        out.append(result("原有选项未被破坏(成功/失败/处理中 仍在)", all(x in choices for x in ("成功", "失败", "处理中")), "choices=%s" % choices))

    return out


def main():
    iteration = sys.argv[1]
    tag = sys.argv[2]
    iteration_dir = ROOT / iteration
    for eval_dir in sorted(iteration_dir.glob("eval-*")):
        ev_name = eval_dir.name.split("-", 2)[2]
        for config_dir in sorted(p for p in eval_dir.iterdir() if p.is_dir()):
            run_dir = config_dir / "run-1"
            if not (run_dir / "pi.jsonl").exists():
                continue
            cfg = env_all.get(config_dir.name) or env_all['with_skill']
            cfg_short = "W" if config_dir.name == "with_skill" else "O"
            tag_cfg = f"{tag}{cfg_short}"
            try:
                expectations = grade_eval(cfg, ev_name, tag_cfg, run_dir, cfg_short)
            except Exception as e:
                expectations = [result("判分异常", False, repr(e))]
            g = {"expectations": expectations}
            (run_dir / "grading.json").write_text(json.dumps(g, indent=2, ensure_ascii=False))
            npass = sum(1 for x in expectations if x["passed"])
            print(f"{eval_dir.name:38} {config_dir.name:12} {npass}/{len(expectations)}", flush=True)


if __name__ == "__main__":
    main()