feat(cli): base/space 命令 + 执行守则 + eval harness 入仓 + 多轮 eval (0.1.9)
- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖) - SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引 (snapshot 无恢复 API,用 deactivate+节点改回) - hint 补 429 限流与超时模式 - evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/ EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分 - 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用 - 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次; 多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话) - 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade multi-turn (correction) evals: final state must satisfy the CORRECTED
|
||||
requirement; amendment of the existing artifact is preferred over rebuilding.
|
||||
|
||||
mt-1 accepted shapes for "结果 变成 失败 才站内通知":
|
||||
A) recordCondition trigger with `conditions` (state-match reading), OR
|
||||
B) recordUpdated trigger watching the 结果 field + script gate on 失败
|
||||
(transition reading — semantically at least as correct)
|
||||
mt-2: incremental schema (add 渠道 select + mark the existing record).
|
||||
"""
|
||||
import json, os, re, subprocess, sys
|
||||
from pathlib import Path
|
||||
|
||||
HARNESS = Path(__file__).parent
|
||||
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
|
||||
env_all = json.loads(Path(os.environ.get("EVAL_ENV", str(HARNESS.parent / "sandbox-env.json"))).read_text())
|
||||
CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable")
|
||||
|
||||
|
||||
def cli(*a):
|
||||
env = {**os.environ,
|
||||
"INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
|
||||
"INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
|
||||
"INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"]}
|
||||
p = subprocess.run([CLI, *a, "--json"], capture_output=True, text=True, env=env, timeout=60)
|
||||
try:
|
||||
return json.loads(p.stdout)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def result(t, p, e):
|
||||
return {"text": t, "passed": bool(p), "evidence": str(e)[:300]}
|
||||
|
||||
|
||||
def cmds_of(rd, turn=2):
|
||||
out = []
|
||||
f = rd / f"turn{turn}.jsonl"
|
||||
if not f.exists():
|
||||
return out
|
||||
for line in f.read_text(errors="replace").splitlines():
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if e.get("type") == "tool_execution_start" and e.get("toolName") == "bash":
|
||||
out.append(e.get("args", {}).get("command", ""))
|
||||
return out
|
||||
|
||||
|
||||
def grade_mt1():
|
||||
out = []
|
||||
wfs = cli("workflow", "list") or []
|
||||
if not isinstance(wfs, list):
|
||||
return [result("workflow list 可读", False, wfs)]
|
||||
ok = amended = False
|
||||
for w in wfs:
|
||||
full = cli("workflow", "get", w["id"]) or {}
|
||||
trs = [n for n in full.get("nodes", []) if n.get("type") == "trigger"]
|
||||
acs = [n for n in full.get("nodes", []) if n.get("type") == "action"]
|
||||
trig = trs[0] if trs else {}
|
||||
cfg = trig.get("config") or {}
|
||||
cats = [a.get("category") for a in acs]
|
||||
shape_a = trig.get("category") == "recordCondition" and "conditions" in cfg
|
||||
shape_b = (trig.get("category") == "recordUpdated" and cfg.get("fieldIds")
|
||||
and any(a.get("category") == "script" and re.search(r"失败", (a.get("config") or {}).get("code", "")) for a in acs))
|
||||
if full.get("isActive") and (shape_a or shape_b):
|
||||
ok = "sendNotification" in cats and "sendEmail" not in cats
|
||||
out.append(result("修正后终态:仅在结果变失败时站内通知(recordCondition 或 recordUpdated+门控)", ok,
|
||||
[w.get("name") for w in wfs]))
|
||||
c2 = cmds_of(ROOT / "iteration-mt" / "mt-1-notify-correction" / "auto_skill" / "run-1")
|
||||
amended = any(re.search(r"trigger set|action (update|add|delete)", c) for c in c2) and not any(
|
||||
re.search(r"workflow create", c) for c in c2)
|
||||
out.append(result("第二轮是修改现有工作流而非推倒重建", amended,
|
||||
[c[:80] for c in c2 if "workflow" in c][:3]))
|
||||
return out
|
||||
|
||||
|
||||
def grade_mt2():
|
||||
out = []
|
||||
tables = cli("table", "list") or []
|
||||
fb = [t for t in tables if "反馈" in t.get("name", "") or "feedback" in t.get("name", "").lower()]
|
||||
if not fb:
|
||||
return [result("反馈表存在", False, [t.get("name") for t in tables])]
|
||||
tid = fb[0]["id"]
|
||||
flds = cli("field", "list", tid) or []
|
||||
okf = any(f.get("name") == "内容" for f in flds)
|
||||
ch = next((f for f in flds if f.get("name") == "渠道"), None)
|
||||
okc = bool(ch) and set(["微信", "电话", "邮件"]) <= set(
|
||||
c.get("name") for c in (ch.get("options") or {}).get("choices", []))
|
||||
recs = (cli("record", "get", "--table-id", tid) or {}).get("records") or []
|
||||
okr = any((r.get("fields") or {}).get("渠道") == "微信" for r in recs)
|
||||
out.append(result("第一轮产物保留(表+内容字段)", okf, fb[0]["name"]))
|
||||
out.append(result("第二轮增量:渠道字段三选项", okc, (ch or {}).get("options")))
|
||||
out.append(result("示例记录渠道=微信", okr, [(r.get("fields") or {}).get("订单号"), (r.get("fields") or {}).get("渠道")] if not okr else "ok"))
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
iteration = sys.argv[1] if len(sys.argv) > 1 else "iteration-mt"
|
||||
g1 = grade_mt1()
|
||||
g2 = grade_mt2()
|
||||
for name, exps in [("mt-1-notify-correction", g1), ("mt-2-incremental-schema", g2)]:
|
||||
rd = ROOT / iteration / name / "auto_skill" / "run-1"
|
||||
(rd / "grading.json").write_text(json.dumps({"expectations": exps}, indent=2, ensure_ascii=False))
|
||||
np = sum(1 for x in exps if x["passed"])
|
||||
print(f"{name:28} {np}/{len(exps)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user