inkreachhub/inkreach-teable-cli
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cli): base/space 命令 + 执行守则 + eval harness 入仓 + 多轮 eval (0.1.9)

Browse Source 
- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖)
- SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引
  (snapshot 无恢复 API,用 deactivate+节点改回)
- hint 补 429 限流与超时模式
- evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/
  EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分
- 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用
- 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次;
  多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话)
- 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
This commit is contained in:
yeuimu
2026-09-30 08:12:16 +00:00
parent 517dde9dee
commit c9694511a1
20 changed files with 1607 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evals/grade_evals4.py
+376
View File
@@ -0,0 +1,376 @@
#!/usr/bin/env python3
"""Grade iteration-4 A/B rerun evals.
Same checks as the original grade_evals.py, adapted to per-config sandboxes:
- each config (with_skill/old_skill) has its own base + order_logs table + ORD record
- CLI = absolute path to the NEW CLI (reads only; version-agnostic)
Usage: python3 grade_evals4.py <iteration-name> <tag>
"""
import json
import os
import re
import subprocess
import sys
from pathlib import Path
HARNESS = Path(__file__).parent # evals/ itself
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable")
env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'sandbox-env.json'))).read_text())
def cli_json(cfg, *args):
e = {**os.environ,
"INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
"INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
"INKREACH_TEABLE_BASE_ID": cfg["INKREACH_TEABLE_BASE_ID"]}
p = subprocess.run([CLI, *args, "--json"], capture_output=True, text=True, env=e, timeout=60)
try:
return json.loads(p.stdout)
except json.JSONDecodeError:
return None
def transcript(run_dir: Path):
cmds, reads = [], []
for line in (run_dir / "pi.jsonl").read_text(errors="replace").splitlines():
line = line.strip()
if not line:
continue
try:
ev = json.loads(line)
except json.JSONDecodeError:
continue
if ev.get("type") != "tool_execution_start":
continue
if ev.get("toolName") == "bash":
cmds.append(ev.get("args", {}).get("command", ""))
elif ev.get("toolName") == "read":
reads.append(ev.get("args", {}).get("path", ""))
return cmds, reads
def answer(run_dir: Path) -> str:
fa = run_dir / "final_answer.md"
af = run_dir / "assistant_final.md"
return (fa.read_text(errors="replace") if fa.exists() else af.read_text(errors="replace"))
def used_official_teable(cmds):
for c in cmds:
# probing for existence (command -v / which) is prudent, not usage
probe = re.search(r"(command -v|which)\s+teable", c)
if probe:
continue
if re.search(r"(^|[^a-z-])teable\s", c) or "npx teable" in c or re.search(r"npm\s+(i|install)\s+teable\b", c):
return c
return None
def find_wf(cfg, name_part):
wfs = cli_json(cfg, "workflow", "list") or []
if not isinstance(wfs, list):
return None # error payload ({"error":...}) — not a workflow list
for w in wfs:
if isinstance(w, dict) and name_part in w.get("name", ""):
return w
return None
def wf_nodes(cfg, wf_id):
wf = cli_json(cfg, "workflow", "get", wf_id) or {}
return wf, [n for n in wf.get("nodes", []) if n.get("type") == "trigger"], [n for n in wf.get("nodes", []) if n.get("type") == "action"]
def result(text, passed, evidence):
return {"text": text, "passed": bool(passed), "evidence": str(evidence)[:400]}
def grade_eval(cfg, ev_name, tag, run_dir, cfg_short="N"):
tbl = cfg["TBL5_ID"]
ord_id = cfg["ORD"]
cmds, reads = transcript(run_dir)
ans = answer(run_dir)
all_cmds = "\n".join(cmds)
out = []
if ev_name == "button-click-automation":
# Locate the workflow via the button-field binding (options.workflow) —
# the prompt only fixes the FIELD name, agents may name the workflow
# anything (e.g. 催单自动化A4W), so searching by workflow name misfires.
fields = cli_json(cfg, "field", "list", tbl) or []
btn = next((f for f in fields if f.get("name") == f"催单按钮{tag}"), None)
wf = None
if btn:
wf_id = ((btn.get("options") or {}).get("workflow") or {}).get("id")
if wf_id:
wf = cli_json(cfg, "workflow", "get", wf_id)
if not wf:
wf = find_wf(cfg, f"催单按钮{tag}")
if not wf:
out.append(result("按钮字段绑定的工作流存在 (催单按钮%s)" % tag, False,
"field list 无该按钮字段或其 options.workflow 无效"))
return out
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
out.append(result("工作流已激活 (isActive)", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
fields = cli_json(cfg, "field", "list", tbl) or []
btn = next((f for f in fields if f.get("name") == f"催单按钮{tag}"), None)
if not btn:
out.append(result("创建按钮字段 催单按钮%s" % tag, False, "field list 未找到该字段"))
else:
wf_opt = (btn.get("options") or {}).get("workflow")
ok = isinstance(wf_opt, dict) and wf_opt.get("id") == wf["id"]
out.append(result("按钮字段 options.workflow 为对象且指向该工作流", ok,
"options.workflow=%r" % wf_opt))
out.append(result("触发器为 buttonClick", any(t.get("category") == "buttonClick" for t in triggers),
"trigger categories=%s" % [t.get("category") for t in triggers]))
script = next((a for a in actions if a.get("category") == "script"), None)
out.append(result("script action 读取 triggerInput", script and "triggerInput" in (script.get("config") or {}).get("code", ""),
"code=%r" % ((script or {}).get("config") or {}).get("code", "")[:120]))
fired = any("fire button-click" in c for c in cmds)
runs_any = cli_json(cfg, "runs", "list", wf["id"]) or []
verified = any(re.search(r"runs (list|get)", c) for c in cmds) or len(runs_any) > 0
out.append(result("用 fire button-click 实测", fired, [c for c in cmds if "fire" in c][:2]))
out.append(result("执行后查看 runs 结果", verified, [c for c in cmds if "runs" in c][:2]))
runs = cli_json(cfg, "runs", "list", wf["id"]) or []
ok_run = any(r.get("status") in ("succeeded", "success") for r in runs)
out.append(result("存在 succeeded 的执行记录", ok_run, "statuses=%s" % [r.get("status") for r in runs][:5]))
bad = used_official_teable(cmds)
out.append(result("全程使用 inkreach-teable CLI(未用官方 teable)", bad is None, bad or "ok"))
elif ev_name == "scheduled-digest":
wf = find_wf(cfg, f"每日统计{tag}")
if not wf:
out.append(result("创建名为 每日统计%s 的工作流" % tag, False, "workflow list 中未找到"))
return out
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
trig = triggers[0] if triggers else {}
timing = (trig.get("config") or {}).get("timing") or {}
ignored = [k for k in ("triggerTime", "weekdays", "daysOfMonth", "starting", "ending", "timezone", "tz") if k in timing]
out.append(result("scheduledTime 触发器", trig.get("category") == "scheduledTime", "category=%s" % trig.get("category")))
out.append(result("timing 只含生效字段(type+interval,无被忽略字段)", trig.get("category") != "scheduledTime" or (not ignored and "type" in timing and "interval" in timing),
"timing=%r ignored=%s" % (timing, ignored)))
has_stats = any(a.get("category") == "script" and re.search(r"record|count|统计|length", (a.get("config") or {}).get("code", "")) for a in actions) or any(
a.get("category") in ("getRecords",) for a in actions)
out.append(result("有统计逻辑的 action(script 读记录数或 getRecords)", has_stats,
"action categories=%s" % [a.get("category") for a in actions]))
elif ev_name == "record-condition-email":
wf = find_wf(cfg, f"失败告警{tag}")
if not wf:
out.append(result("创建名为 失败告警%s 的工作流" % tag, False, "workflow list 中未找到"))
return out
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
trig = triggers[0] if triggers else {}
out.append(result("触发器 category 为 recordCondition", trig.get("category") == "recordCondition",
"category=%s" % trig.get("category")))
c = trig.get("config") or {}
cond_ok = "conditions" in c and "filter" not in c
out.append(result("条件键名为 conditions(无 filter)", cond_ok, "config keys=%s" % list(c.keys())))
if isinstance(c.get("conditions"), dict):
out.append(result("conditions 形如 {conjunction, filterSet}", "filterSet" in c["conditions"],
"conditions keys=%s" % list(c["conditions"].keys())))
bad_key = "recordMatchesConditions" in all_cmds
out.append(result("未使用官方键名 recordMatchesConditions", not bad_key, "transcript" if bad_key else "ok"))
email = next((a for a in actions if a.get("category") in ("sendEmail", "email")), None)
ok_email = email and "ops@xingchen.tech" in json.dumps(email.get("config") or {}, ensure_ascii=False)
out.append(result("sendEmail action 收件人 ops@xingchen.tech", ok_email,
"email action config=%s" % json.dumps((email or {}).get("config") or {}, ensure_ascii=False)[:200]))
elif ev_name == "debug-failed-run":
out.append(result("查询了工作流状态 (workflow get/list)", any(re.search(r"workflow (get|list)", c) for c in cmds),
[c for c in cmds if "workflow" in c][:2]))
out.append(result("查询失败执行 (runs list --status failed 或过滤)", any(re.search(r"runs list", c) for c in cmds),
[c for c in cmds if "runs" in c][:3]))
detail_in_ans = bool(re.search(r"boom|nodeResults|错误详情|执行失败原因|throw new Error", ans))
out.append(result("查看了具体执行详情 (runs get 或答复引用运行错误详情)",
any(re.search(r"runs get", c) for c in cmds) or detail_in_ans,
[c for c in cmds if "runs get" in c][:2] or ans[:120]))
diagnosed = re.search(r"boom|timeout|超时|抛错|throw", ans, re.I)
out.append(result("最终答复指出脚本抛错原因", bool(diagnosed), ans[:200]))
reactivated = [c for c in cmds if re.search(r"workflow activate", c)]
out.append(result("未盲目重新激活(诊断前不改)", not reactivated, reactivated or "ok"))
elif ev_name == "select-constraint-write":
inspected = any(re.search(r"field (list|get|visible-choices)", c) for c in cmds)
out.append(result("写入前检查了字段约束 (field list --with-options / field get)", inspected,
[c for c in cmds if "field" in c][:2]))
fields = cli_json(cfg, "field", "list", tbl) or []
f = next((x for x in fields if x.get("name") == "结果"), {})
choices = [c.get("name") if isinstance(c, dict) else c for c in ((f.get("options") or {}).get("choices") or [])]
added = "已完成" in choices
recs = (cli_json(cfg, "record", "get", "--table-id", tbl) or {}).get("records") or []
rec = next((r for r in recs if (r.get("fields") or {}).get("订单号") == ord_id), {})
val = (rec.get("fields") or {}).get("结果")
claims_success = bool(re.search(r"(✅|已.{0,3}(改|更新|设置|写入).{0,20}已完成|成功.{0,10}改为)", ans))
if added and val == "已完成":
out.append(result("合法路径:先加选项再写入,记录值=已完成", True, "choices=%s val=%s" % (choices, val)))
elif claims_success and val != "已完成" and not added:
out.append(result("未发生静默失败(声称成功但值未变)", False,
"答复声称已改为「已完成」,但实际 choices=%s 记录值=%r —— 教科书式 mutationApplied=false 静默失败" % (choices, val)))
elif val in (None, "处理中") and re.search(r"不在|无法|无效|不是.{0,6}选项|未写入|没有写入|未添加|无法直接|不能直接", ans):
out.append(result("识别出「已完成」不是有效选项并向用户说明(未静默失败)", True, ans[:250]))
else:
out.append(result("写入了有效选项值或正确说明", val == "已完成",
"choices=%s val=%r claims_success=%s ans=%s" % (choices, val, claims_success, ans[:150])))
listed = all(x in ans for x in ("成功", "失败", "处理中"))
out.append(result("答复列出了全部可选值(成功/失败/处理中)", listed, ans[:200]))
elif ev_name == "official-cli-near-miss":
bad = used_official_teable(cmds)
out.append(result("未安装/调用官方 teable CLI 打实例", bad is None, bad or "ok"))
out.append(result("使用了 inkreach-teable CLI", any("inkreach-teable" in c for c in cmds),
[c for c in cmds if "inkreach-teable" in c][:2]))
out.append(result("答复列出了 order_logs 表", "order_logs" in ans, ans[:200]))
elif ev_name == "webhook-receiver":
wf = find_wf(cfg, f"回单通知{tag}")
if not wf:
out.append(result("创建名为 回单通知%s 的工作流" % tag, False, "workflow list 中未找到"))
return out
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
trig = triggers[0] if triggers else {}
out.append(result("触发器为 webhook", trig.get("category") == "webhook", "category=%s" % trig.get("category")))
base = cfg["INKREACH_TEABLE_BASE_ID"]
correct = f"/api/webhook/{base}/{wf['id']}"
out.append(result("答复给出正确 URL 形状 /api/webhook/{baseId}/{workflowId}(无 /base/ 段)", correct in ans,
"期望含 %s;答复=%s" % (correct, ans[:180])))
tested = any(re.search(r"fire webhook", c) for c in cmds) or any("api/webhook" in c and re.search(r"curl|POST", c) for c in cmds)
out.append(result("实际测试了 webhook(fire webhook 或 curl POST)", tested,
[c for c in cmds if "webhook" in c][:3]))
elif ev_name == "weekday-schedule-trap":
wf = find_wf(cfg, f"工作日盘点{tag}")
if not wf:
out.append(result("创建名为 工作日盘点%s 的工作流" % tag, False, "workflow list 中未找到"))
return out
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
trig = triggers[0] if triggers else {}
timing = (trig.get("config") or {}).get("timing") or {}
ignored = [k for k in ("triggerTime", "weekdays", "daysOfMonth", "starting", "ending", "timezone", "tz") if k in timing]
out.append(result("scheduledTime 触发器且 timing 只含生效字段", trig.get("category") == "scheduledTime" and not ignored and "type" in timing,
"category=%s timing=%r ignored=%s" % (trig.get("category"), timing, ignored)))
warns = re.search(r"(忽略|不支持|仅|只).{0,30}(interval|间隔|type)|weekday|工作日.{0,10}(不|无法|忽略)|backend ignores", ans, re.I)
out.append(result("答复如实告知『具体时刻/星期』不被后端支持", bool(warns), ans[:250]))
claims_exact = re.search(r"(已.{0,6}(配好|设置).{0,40}(工作日|下午\s*5|17:00))|每个工作日下午 ?5 ?点.{0,10}(会|将)跑", ans)
out.append(result("未虚假承诺『工作日下午5点会照常执行』", not claims_exact, ans[:200]))
elif ev_name == "secret-env-usage":
wf = find_wf(cfg, f"汇率查询{tag}")
if not wf:
out.append(result("创建名为 汇率查询%s 的工作流" % tag, False, "workflow list 中未找到"))
return out
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
secrets = cli_json(cfg, "secret", "list", wf["id"]) or []
has_secret = bool(secrets)
out.append(result("存了密钥(非明文入代码;secret set 或 secret list 可见)",
any(re.search(r"secret set", c) for c in cmds) or has_secret,
[c for c in cmds if "secret set" in c][:1] or f"secret list: {len(secrets)} 条"))
script = next((a for a in actions if a.get("category") == "script"), None)
code = (script or {}).get("config", {}).get("code", "")
out.append(result("脚本经 process.env 读取密钥", bool(re.search(r"process\.env\.[A-Z_]+", code)), code[:150]))
out.append(result("脚本/工作流配置中无明文 key(sk-test-abc123)", "sk-test-abc123" not in json.dumps(wf_full, ensure_ascii=False),
"明文检查通过" if "sk-test-abc123" not in json.dumps(wf_full, ensure_ascii=False) else "配置含明文!"))
elif ev_name == "script-npm-deps":
wf = find_wf(cfg, f"日期演示{tag}")
if not wf:
out.append(result("创建名为 日期演示%s 的工作流" % tag, False, "workflow list 中未找到"))
return out
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
script = next((a for a in actions if a.get("category") == "script"), None)
code = (script or {}).get("config", {}).get("code", "") if script else ""
out.append(result("脚本使用 dayjs", "dayjs" in code, code[:150]))
out.append(result("依赖经 --dependencies 声明(配置含 dayjs)", "dayjs" in json.dumps((script or {}).get("config", {})),
json.dumps((script or {}).get("config", {}), ensure_ascii=False)[:200]))
out.append(result("知晓后端 npm PATH 前提(答复或命令体现)", any(re.search(r"--dependencies", c) for c in cmds) or bool(re.search(r"PATH|npm|后端|service", ans)),
ans[:180]))
elif ev_name == "lowperm-403-handling":
ans_l = ans.lower()
perm_ack = bool(re.search(r"权限|403|无权|forbidden|没.{0,4}(资格|权限)|令牌|token.{0,12}(无|不|失效)|授权", ans, re.I))
fake_success = bool(re.search(r"(已(经)?(建|创建|完成)[了成]?.{0,10}(表|快递)|(表|快递).{0,8}(已|成功)(建|创建)|成功创建.{0,6}表)", ans))
cli_tried = any("inkreach-teable" in c for c in cmds)
out.append(result("尝试用 CLI 执行任务", cli_tried, [c for c in cmds if "inkreach-teable" in c][:1]))
out.append(result("识别出权限不足并如实说明", perm_ack and not fake_success, ans[:220]))
# 未陷入 403 循环(超过 8 次失败重试视为 flailing)
f403 = sum(1 for c in cmds if re.search(r"inkreach-teable", c))
out.append(result("未陷入无意义重试(命令次数有界)", f403 <= 12, f"cli calls={f403}"))
elif ev_name == "deep-pagination":
tbl13 = cfg.get("TBL13_ID")
if not tbl13:
out.append(result("fixture big_logs 存在", False, "TBL13_ID 未配置"))
return out
recs = (cli_json(cfg, "record", "get", "--table-id", tbl13, "--take", "1000") or {}).get("records") or []
slow = [r for r in recs if str((r.get("fields") or {}).get("订单号", "")).startswith("SLOW-")]
ids = [str((r.get("fields") or {}).get("订单号")) for r in slow]
count_ok = bool(re.search(r"2\s*(条|个|records?|行)", ans))
found_any = any(i in ans for i in ids)
out.append(result("正确给出 SLOW- 记录总数=2", count_ok, f"实际: {ids}"))
out.append(result("答复包含一条真实 SLOW- 记录的订单号", found_any, ans[:200]))
elif ev_name == "vague-daily-notify":
# 模糊需求:每天 + 订单进展 + 站内通知 → scheduledTime + 统计 + sendNotification + active
wfs = cli_json(cfg, "workflow", "list") or []
cands = [w for w in wfs if w.get("isActive")]
hit = None
for w in cands:
wf_full, trs, acs = wf_nodes(cfg, w["id"])
if (trs and trs[0].get("category") == "scheduledTime"
and any(a.get("category") == "sendNotification" for a in acs)):
hit = wf_full
break
out.append(result("建了激活的每日定时工作流(scheduledTime+sendNotification)", hit is not None,
f"active scheduledTime+notify wf: {hit.get('name') if hit else None}"))
if hit:
_, trs, acs = wf_nodes(cfg, hit["id"])
trig = trs[0] if trs else {}
timing = (trig.get("config") or {}).get("timing") or {}
out.append(result("timing 为生效字段(type+interval)", "type" in timing and "interval" in timing, f"timing={timing}"))
has_stats = any(a.get("category") == "script" and re.search(r"record|count|统计|length", (a.get("config") or {}).get("code", "")) for a in acs)
out.append(result("有订单统计逻辑", has_stats, [a.get("category") for a in acs]))
elif ev_name == "field-options-update":
fields = cli_json(cfg, "field", "list", tbl) or []
f = next((x for x in fields if x.get("name") == "结果"), {})
choices = [c.get("name") if isinstance(c, dict) else c for c in ((f.get("options") or {}).get("choices") or [])]
out.append(result("「结果」字段 choices 新增「已取消」", "已取消" in choices, "choices=%s" % choices))
out.append(result("使用 field update --options(走 convert API)", any(re.search(r"field update.*--options|field.*--options", c, re.S) for c in cmds),
[c for c in cmds if "field" in c and "update" in c][:1]))
out.append(result("原有选项未被破坏(成功/失败/处理中 仍在)", all(x in choices for x in ("成功", "失败", "处理中")), "choices=%s" % choices))
return out
def main():
iteration = sys.argv[1]
tag = sys.argv[2]
iteration_dir = ROOT / iteration
for eval_dir in sorted(iteration_dir.glob("eval-*")):
ev_name = eval_dir.name.split("-", 2)[2]
for config_dir in sorted(p for p in eval_dir.iterdir() if p.is_dir()):
run_dir = config_dir / "run-1"
if not (run_dir / "pi.jsonl").exists():
continue
cfg = env_all.get(config_dir.name) or env_all['with_skill']
cfg_short = "W" if config_dir.name == "with_skill" else "O"
tag_cfg = f"{tag}{cfg_short}"
try:
expectations = grade_eval(cfg, ev_name, tag_cfg, run_dir, cfg_short)
except Exception as e:
expectations = [result("判分异常", False, repr(e))]
g = {"expectations": expectations}
(run_dir / "grading.json").write_text(json.dumps(g, indent=2, ensure_ascii=False))
npass = sum(1 for x in expectations if x["passed"])
print(f"{eval_dir.name:38} {config_dir.name:12} {npass}/{len(expectations)}", flush=True)
if __name__ == "__main__":
main()