feat(cli): base/space 命令 + 执行守则 + eval harness 入仓 + 多轮 eval (0.1.9)

- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖)
- SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引
  (snapshot 无恢复 API,用 deactivate+节点改回)
- hint 补 429 限流与超时模式
- evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/
  EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分
- 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用
- 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次;
  多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话)
- 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
This commit is contained in:
yeuimu
2026-09-30 08:12:16 +00:00
parent 517dde9dee
commit c9694511a1
20 changed files with 1607 additions and 9 deletions
+111
View File
@@ -0,0 +1,111 @@
#!/usr/bin/env python3
"""Grade multi-turn (correction) evals: final state must satisfy the CORRECTED
requirement; amendment of the existing artifact is preferred over rebuilding.
mt-1 accepted shapes for "结果 变成 失败 才站内通知":
A) recordCondition trigger with `conditions` (state-match reading), OR
B) recordUpdated trigger watching the 结果 field + script gate on 失败
(transition reading — semantically at least as correct)
mt-2: incremental schema (add 渠道 select + mark the existing record).
"""
import json, os, re, subprocess, sys
from pathlib import Path
HARNESS = Path(__file__).parent
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
env_all = json.loads(Path(os.environ.get("EVAL_ENV", str(HARNESS.parent / "sandbox-env.json"))).read_text())
CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable")
def cli(*a):
env = {**os.environ,
"INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
"INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
"INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"]}
p = subprocess.run([CLI, *a, "--json"], capture_output=True, text=True, env=env, timeout=60)
try:
return json.loads(p.stdout)
except json.JSONDecodeError:
return None
def result(t, p, e):
return {"text": t, "passed": bool(p), "evidence": str(e)[:300]}
def cmds_of(rd, turn=2):
out = []
f = rd / f"turn{turn}.jsonl"
if not f.exists():
return out
for line in f.read_text(errors="replace").splitlines():
try:
e = json.loads(line)
except json.JSONDecodeError:
continue
if e.get("type") == "tool_execution_start" and e.get("toolName") == "bash":
out.append(e.get("args", {}).get("command", ""))
return out
def grade_mt1():
out = []
wfs = cli("workflow", "list") or []
if not isinstance(wfs, list):
return [result("workflow list 可读", False, wfs)]
ok = amended = False
for w in wfs:
full = cli("workflow", "get", w["id"]) or {}
trs = [n for n in full.get("nodes", []) if n.get("type") == "trigger"]
acs = [n for n in full.get("nodes", []) if n.get("type") == "action"]
trig = trs[0] if trs else {}
cfg = trig.get("config") or {}
cats = [a.get("category") for a in acs]
shape_a = trig.get("category") == "recordCondition" and "conditions" in cfg
shape_b = (trig.get("category") == "recordUpdated" and cfg.get("fieldIds")
and any(a.get("category") == "script" and re.search(r"失败", (a.get("config") or {}).get("code", "")) for a in acs))
if full.get("isActive") and (shape_a or shape_b):
ok = "sendNotification" in cats and "sendEmail" not in cats
out.append(result("修正后终态:仅在结果变失败时站内通知(recordCondition 或 recordUpdated+门控)", ok,
[w.get("name") for w in wfs]))
c2 = cmds_of(ROOT / "iteration-mt" / "mt-1-notify-correction" / "auto_skill" / "run-1")
amended = any(re.search(r"trigger set|action (update|add|delete)", c) for c in c2) and not any(
re.search(r"workflow create", c) for c in c2)
out.append(result("第二轮是修改现有工作流而非推倒重建", amended,
[c[:80] for c in c2 if "workflow" in c][:3]))
return out
def grade_mt2():
out = []
tables = cli("table", "list") or []
fb = [t for t in tables if "反馈" in t.get("name", "") or "feedback" in t.get("name", "").lower()]
if not fb:
return [result("反馈表存在", False, [t.get("name") for t in tables])]
tid = fb[0]["id"]
flds = cli("field", "list", tid) or []
okf = any(f.get("name") == "内容" for f in flds)
ch = next((f for f in flds if f.get("name") == "渠道"), None)
okc = bool(ch) and set(["微信", "电话", "邮件"]) <= set(
c.get("name") for c in (ch.get("options") or {}).get("choices", []))
recs = (cli("record", "get", "--table-id", tid) or {}).get("records") or []
okr = any((r.get("fields") or {}).get("渠道") == "微信" for r in recs)
out.append(result("第一轮产物保留(表+内容字段)", okf, fb[0]["name"]))
out.append(result("第二轮增量:渠道字段三选项", okc, (ch or {}).get("options")))
out.append(result("示例记录渠道=微信", okr, [(r.get("fields") or {}).get("订单号"), (r.get("fields") or {}).get("渠道")] if not okr else "ok"))
return out
def main():
iteration = sys.argv[1] if len(sys.argv) > 1 else "iteration-mt"
g1 = grade_mt1()
g2 = grade_mt2()
for name, exps in [("mt-1-notify-correction", g1), ("mt-2-incremental-schema", g2)]:
rd = ROOT / iteration / name / "auto_skill" / "run-1"
(rd / "grading.json").write_text(json.dumps({"expectations": exps}, indent=2, ensure_ascii=False))
np = sum(1 for x in exps if x["passed"])
print(f"{name:28} {np}/{len(exps)}")
if __name__ == "__main__":
main()