feat(cli): base/space 命令 + 执行守则 + eval harness 入仓 + 多轮 eval (0.1.9)
- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖) - SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引 (snapshot 无恢复 API,用 deactivate+节点改回) - hint 补 429 限流与超时模式 - evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/ EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分 - 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用 - 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次; 多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话) - 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
This commit is contained in:
yeuimu
@@ -0,0 +1,25 @@
|
|||||||
|
# Skill/CLI eval harness
|
||||||
|
|
||||||
|
14 题 teable 操作 eval + 6 题 plugin eval + 逐次尝试统计。单源真值在本目录,
|
||||||
|
运行产物默认落 `/root/eval-rerun`(可用 `EVAL_ROOT` 覆盖)。
|
||||||
|
|
||||||
|
## 目录
|
||||||
|
- `teable-evals.json` / `plugin-evals.json` — 题目(prompt 模板,支持 {TAG}/{WFID}/{TBL5}/{ORD}/{TBL})
|
||||||
|
- `run_evals4.py` — 启动器。支持配置: `with_skill=<skill>:<bin>`, `old_skill=...`, `auto_skill=NONE:<bin>`(触发模式:不给技能路径,靠用户级技能自动发现);`EVAL_MODEL` 换模型
|
||||||
|
- `grade_evals4.py` / `grade_plugin_evals.py` — 判分(按最终状态优先)。重判历史轮次须冻结该轮 env(`EVAL_ENV`)
|
||||||
|
- `run_multiturn.py` + `grade_multiturn.py` — 两轮修正式对话 eval(pi `--session-id` 续接;判分认 recordCondition 或 recordUpdated+门控两种正确形状)
|
||||||
|
- `analyze_attempts.py ROOT...` — 尝试次数分布(第几次成功/never;`EVAL_CFG` 选配置目录)
|
||||||
|
- `analyze_trajectories.py` — 轨迹解析公用库(失败签名分类器)
|
||||||
|
|
||||||
|
## 一次完整 A/B
|
||||||
|
```bash
|
||||||
|
EVAL_ROOT=/root/eval-rerun python3 evals/run_evals4.py iteration-X A1 \
|
||||||
|
"with_skill=/path/skill-new:/path/bin,old_skill=/path/skill-old:/path/bin-old"
|
||||||
|
python3 evals/grade_evals4.py iteration-X A1
|
||||||
|
python3 evals/analyze_attempts.py /root/eval-rerun/iteration-X
|
||||||
|
```
|
||||||
|
|
||||||
|
## 环境要求
|
||||||
|
- pi(opencode-go 凭据)+ 隔离 HOME(无用户级技能,除 auto_skill 模式用带技能副本)
|
||||||
|
- 每轮独立沙箱 base;判分后清理。**一切建库仅在 agent-playground 专用空间**。
|
||||||
|
- eval-12 需低权限 PAT(sandbox-env.json 的 LOWPERM_TOKEN)
|
||||||
Binary file not shown.
@@ -0,0 +1,83 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Attempts-to-success distribution for iteration-5 (N=3 variance rerun).
|
||||||
|
|
||||||
|
For every CLI verb session across all runs: which attempt number succeeded
|
||||||
|
(1 / 2 / 3+ / never)? The user-facing goal is "1-2 attempts per call".
|
||||||
|
"""
|
||||||
|
import json, glob, os, sys
|
||||||
|
from pathlib import Path
|
||||||
|
from collections import defaultdict
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent))
|
||||||
|
from analyze_trajectories import load_run, classify_output, cli_verb
|
||||||
|
|
||||||
|
def sessions_for(pi):
|
||||||
|
calls, meta = load_run(pi)
|
||||||
|
cli = []
|
||||||
|
for tool, cmd, out in calls:
|
||||||
|
if tool != 'bash':
|
||||||
|
continue
|
||||||
|
cli.append((cli_verb(cmd), cmd, classify_output(out, cmd)))
|
||||||
|
# verb -> attempts used until success (per contiguous session)
|
||||||
|
stats = defaultdict(lambda: {'sess': 0, 'a1': 0, 'a2': 0, 'a3p': 0, 'never': 0})
|
||||||
|
cur = defaultdict(lambda: {'fails': 0, 'done': False})
|
||||||
|
for verb, cmd, sig in cli:
|
||||||
|
if not verb:
|
||||||
|
continue
|
||||||
|
st = cur[verb]
|
||||||
|
if sig is None:
|
||||||
|
if not st['done']:
|
||||||
|
stats[verb]['sess'] += 1
|
||||||
|
n = st['fails'] + 1
|
||||||
|
if n == 1: stats[verb]['a1'] += 1
|
||||||
|
elif n == 2: stats[verb]['a2'] += 1
|
||||||
|
else: stats[verb]['a3p'] += 1
|
||||||
|
st['done'] = True; st['fails'] = 0
|
||||||
|
else:
|
||||||
|
if not st['done']:
|
||||||
|
st['fails'] += 1
|
||||||
|
for verb, st in cur.items():
|
||||||
|
if not st['done'] and st['fails']:
|
||||||
|
stats[verb]['sess'] += 1; stats[verb]['never'] += 1
|
||||||
|
return stats
|
||||||
|
|
||||||
|
def main():
|
||||||
|
roots = sys.argv[1:] or ['/root/eval-rerun/iteration-5']
|
||||||
|
cfg_glob = os.environ.get('EVAL_CFG', 'with_skill')
|
||||||
|
tot = {'sess': 0, 'a1': 0, 'a2': 0, 'a3p': 0, 'never': 0}
|
||||||
|
zero = dict(tot)
|
||||||
|
per_verb = defaultdict(lambda: dict(zero))
|
||||||
|
npass = nrun = 0
|
||||||
|
files = sorted(f for r in roots for f in glob.glob(f'{r}/eval-*/{cfg_glob}/run-1/pi.jsonl'))
|
||||||
|
for pi in files:
|
||||||
|
nrun += 1
|
||||||
|
gp = os.path.join(os.path.dirname(pi), 'grading.json')
|
||||||
|
if os.path.exists(gp):
|
||||||
|
try:
|
||||||
|
g = json.load(open(gp)); exps = g.get('expectations') or []
|
||||||
|
if exps and all(x.get('passed') for x in exps):
|
||||||
|
npass += 1
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
for verb, st in sessions_for(pi).items():
|
||||||
|
for k in tot:
|
||||||
|
per_verb[verb][k] += st[k]
|
||||||
|
tot[k] += st[k]
|
||||||
|
print(f"runs={nrun} eval-pass={npass}/{nrun}")
|
||||||
|
s = tot['sess']
|
||||||
|
if s:
|
||||||
|
print(f"\nverb sessions: {s}")
|
||||||
|
print(f" attempt 1: {tot['a1']:4} ({tot['a1']/s*100:.1f}%)")
|
||||||
|
print(f" attempt 2: {tot['a2']:4} ({tot['a2']/s*100:.1f}%)")
|
||||||
|
print(f" attempt 3+: {tot['a3p']:4} ({tot['a3p']/s*100:.1f}%)")
|
||||||
|
print(f" never ok: {tot['never']:4} ({tot['never']/s*100:.1f}%)")
|
||||||
|
print(f" ≤2 attempts: {(tot['a1']+tot['a2'])/s*100:.1f}%")
|
||||||
|
print(f"\nper-verb (only verbs with a retry or never):")
|
||||||
|
for verb, st in sorted(per_verb.items(), key=lambda kv: -(kv[1]['a2']+kv[1]['a3p']+kv[1]['never'])):
|
||||||
|
if st['a2'] or st['a3p'] or st['never']:
|
||||||
|
print(f" {verb:24} sess={st['sess']:3} a1={st['a1']:3} a2={st['a2']} a3+={st['a3p']} never={st['never']}")
|
||||||
|
print(f"\nall verbs:")
|
||||||
|
for verb, st in sorted(per_verb.items(), key=lambda kv: -kv[1]['sess']):
|
||||||
|
print(f" {verb:24} sess={st['sess']:3} a1={st['a1']:3} a2={st['a2']} a3+={st['a3p']} never={st['never']}")
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,222 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Trajectory analyzer for pi-skill-lab eval transcripts.
|
||||||
|
|
||||||
|
Parses pi.jsonl transcripts, reconstructs per-run tool-call sequences,
|
||||||
|
detects failed / retried commands (output-text heuristics — pi's bash tool
|
||||||
|
returns no exit codes), and correlates with grading.json outcomes.
|
||||||
|
|
||||||
|
Outputs:
|
||||||
|
1. per-run summary (effort + failure counts + doc-read order)
|
||||||
|
2. aggregated failure signatures per config
|
||||||
|
3. first-attempt success rate for the core CLI verbs
|
||||||
|
"""
|
||||||
|
import json, glob, os, re, sys
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
# --- failure heuristics on tool OUTPUT text (stdout only in pi) -------------
|
||||||
|
# Strict failure patterns: an output line is a FAILURE only when it looks like
|
||||||
|
# an actual error line — NOT when the word "failed"/"error" appears in normal
|
||||||
|
# status output (e.g. `runs list` printing status=failed, or "error": null).
|
||||||
|
FAIL_LINES = [
|
||||||
|
r'^\s*[✗✘❌]', r'^\s*error\b', r'\bError:', r'Validation error',
|
||||||
|
r'command not found', r'ENOENT', r'ECONNREFUSED', r'ETIMEDOUT',
|
||||||
|
r'unauthorized', r'\bforbidden\b', r'Missing required',
|
||||||
|
r'unknown (option|command)', r'is required',
|
||||||
|
r'HTTP_?[45]\d\d', r'Cannot (GET|POST|PUT|PATCH|DELETE)\b',
|
||||||
|
r'"status":\s*[45]\d\d', r'Invalid (option|value|input|RecordId|List)',
|
||||||
|
r'Unsupported library type', r'spawn npm ENOENT',
|
||||||
|
]
|
||||||
|
FAIL_RE = re.compile('|'.join(FAIL_LINES))
|
||||||
|
# outputs that legitimately contain these words but are NOT failures
|
||||||
|
OK_RE = re.compile(r'^(id|name|baseId|workflowId|triggerType|config|next|note|\s*$)', re.M)
|
||||||
|
|
||||||
|
def classify_output(text, cmd=''):
|
||||||
|
"""Return failure signature (short str) or None if output looks successful."""
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
# `--help` legitimately prints "Usage:" — not a failure
|
||||||
|
is_help = '--help' in cmd
|
||||||
|
head = text[:400]
|
||||||
|
if is_help and re.match(r'\s*(Usage:|inkreach-teable \[options\])', text):
|
||||||
|
return None
|
||||||
|
for ln in text.splitlines():
|
||||||
|
ln = ln.strip()
|
||||||
|
if ln and FAIL_RE.search(ln):
|
||||||
|
if is_help and ln.startswith('Usage:'):
|
||||||
|
continue
|
||||||
|
return ln[:140]
|
||||||
|
return None
|
||||||
|
|
||||||
|
def cmd_of(args):
|
||||||
|
if not isinstance(args, dict):
|
||||||
|
return ''
|
||||||
|
return args.get('command') or args.get('path') or ''
|
||||||
|
|
||||||
|
def load_run(pi_path):
|
||||||
|
"""Return ordered tool calls [(tool, cmd, out_head)], plus meta."""
|
||||||
|
calls, meta = [], {'turns': 0, 'settled': False}
|
||||||
|
starts = {}
|
||||||
|
for line in open(pi_path):
|
||||||
|
try:
|
||||||
|
e = json.loads(line)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
t = e.get('type')
|
||||||
|
if t == 'tool_execution_start':
|
||||||
|
starts[e['toolCallId']] = e
|
||||||
|
elif t == 'tool_execution_end':
|
||||||
|
s = starts.get(e['toolCallId'], {})
|
||||||
|
args = s.get('args', {})
|
||||||
|
r = e.get('result') or {}
|
||||||
|
c = r.get('content')
|
||||||
|
txt = ''
|
||||||
|
if isinstance(c, list):
|
||||||
|
txt = '\n'.join(x.get('text', '') for x in c if isinstance(x, dict))
|
||||||
|
elif isinstance(c, str):
|
||||||
|
txt = c
|
||||||
|
calls.append((e['toolName'], cmd_of(args), txt[:600]))
|
||||||
|
elif t == 'turn_start':
|
||||||
|
meta['turns'] += 1
|
||||||
|
elif t == 'agent_settled':
|
||||||
|
meta['settled'] = True
|
||||||
|
return calls, meta
|
||||||
|
|
||||||
|
def cli_verb(cmd):
|
||||||
|
m = re.search(r'(?:^|\s|&&|;|npx\s+(?:git+\S+\s+)?)inkreach-teable\s+([a-z-]+)(?:\s+([a-z-]+))?', cmd)
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
return '.'.join(x for x in m.groups() if x)
|
||||||
|
|
||||||
|
def analyze(pi_path):
|
||||||
|
calls, meta = load_run(pi_path)
|
||||||
|
run = {
|
||||||
|
'path': pi_path,
|
||||||
|
'total': len(calls),
|
||||||
|
'bash': sum(1 for t, c, o in calls if t == 'bash'),
|
||||||
|
'reads': sum(1 for t, c, o in calls if t == 'read'),
|
||||||
|
'turns': meta['turns'],
|
||||||
|
'settled': meta['settled'],
|
||||||
|
'cli': [], # (verb, cmd, fail_sig_or_None)
|
||||||
|
'other_fail': [], # non-CLI failed bash (npm, curl, node...)
|
||||||
|
'docs': [], # skill files read, in order
|
||||||
|
}
|
||||||
|
seen_verbs = {}
|
||||||
|
for tool, cmd, out in calls:
|
||||||
|
if tool == 'read':
|
||||||
|
if '/skills/' in cmd and cmd.endswith('.md'):
|
||||||
|
run['docs'].append(cmd.split('/skills/')[-1])
|
||||||
|
continue
|
||||||
|
if tool != 'bash':
|
||||||
|
continue
|
||||||
|
sig = classify_output(out, cmd)
|
||||||
|
v = cli_verb(cmd)
|
||||||
|
if v:
|
||||||
|
run['cli'].append((v, cmd, sig))
|
||||||
|
elif sig:
|
||||||
|
run['other_fail'].append((cmd[:100], sig))
|
||||||
|
return run
|
||||||
|
|
||||||
|
def attempts_to_success(cli_calls):
|
||||||
|
"""Per verb: list of (#failed attempts before first success, succeeded?)."""
|
||||||
|
stats = defaultdict(lambda: {'fails_before': [], 'never': 0, 'first_try': 0, 'total_sessions': 0})
|
||||||
|
cur = defaultdict(lambda: {'fails': 0, 'succeeded': False})
|
||||||
|
for verb, cmd, sig in cli_calls:
|
||||||
|
st = cur[verb]
|
||||||
|
if sig is None:
|
||||||
|
if not st['succeeded']:
|
||||||
|
stats[verb]['total_sessions'] += 1
|
||||||
|
stats[verb]['fails_before'].append(st['fails'])
|
||||||
|
if st['fails'] == 0:
|
||||||
|
stats[verb]['first_try'] += 1
|
||||||
|
st['succeeded'] = True
|
||||||
|
st['fails'] = 0
|
||||||
|
else:
|
||||||
|
if not st['succeeded']:
|
||||||
|
st['fails'] += 1
|
||||||
|
# verbs that never succeeded
|
||||||
|
for verb, st in cur.items():
|
||||||
|
if not st['succeeded'] and st['fails']:
|
||||||
|
stats[verb]['total_sessions'] += 1
|
||||||
|
stats[verb]['never'] += 1
|
||||||
|
return stats
|
||||||
|
|
||||||
|
def main():
|
||||||
|
rows = []
|
||||||
|
for ws in ['inkreach-teable-workspace', 'plugin-workspace']:
|
||||||
|
for pi in sorted(glob.glob(f'/root/pi-skill-lab/{ws}/iteration-*/eval-*/*/run-*/pi.jsonl')):
|
||||||
|
parts = pi.split('/')
|
||||||
|
run = analyze(pi)
|
||||||
|
gp = os.path.join(os.path.dirname(pi), 'grading.json')
|
||||||
|
passed = None
|
||||||
|
if os.path.exists(gp):
|
||||||
|
try:
|
||||||
|
g = json.load(open(gp))
|
||||||
|
exps = g.get('expectations') or []
|
||||||
|
if exps:
|
||||||
|
passed = all(x.get('passed') for x in exps)
|
||||||
|
else:
|
||||||
|
passed = g.get('passed')
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
rows.append({
|
||||||
|
'ws': ws.replace('-workspace',''), 'iter': parts[4],
|
||||||
|
'eval': parts[5], 'cfg': parts[6], 'run': parts[7].replace('run-',''),
|
||||||
|
'passed': passed, **{k: run[k] for k in ('total','bash','reads','turns','settled','cli','other_fail','docs')},
|
||||||
|
})
|
||||||
|
# ---- 1. per-run summary ----
|
||||||
|
print('=' * 110)
|
||||||
|
print('PER-RUN TRAJECTORY SUMMARY')
|
||||||
|
print('=' * 110)
|
||||||
|
hdr = f"{'eval':38} {'cfg':12} {'ok':4} {'tcs':4} {'rd':3} {'trn':4} {'cli':4} {'f':3} {'retry':5} docs"
|
||||||
|
print(hdr)
|
||||||
|
for r in rows:
|
||||||
|
fails = sum(1 for v,c,s in r['cli'] if s)
|
||||||
|
retries = sum(1 for v,c,s in r['cli'] if s) # failed call == at least one retry if task later passed
|
||||||
|
cli_ok = 'Y' if r['cli'] else '-'
|
||||||
|
print(f"{r['eval'][:38]:38} {r['cfg']:12} {str(r['passed'])[:3]:4} {r['total']:4} {r['reads']:3} {r['turns']:4} "
|
||||||
|
f"{len(r['cli']):4} {fails:3} {retries:5} {len(r['docs'])}d {'>' if not r['settled'] else ''}")
|
||||||
|
# ---- 2. failure signatures ----
|
||||||
|
print()
|
||||||
|
print('=' * 110)
|
||||||
|
print('FAILURE SIGNATURES (failed CLI / other-bash calls, per config)')
|
||||||
|
print('=' * 110)
|
||||||
|
sigs = defaultdict(list)
|
||||||
|
for r in rows:
|
||||||
|
for verb, cmd, sig in r['cli']:
|
||||||
|
if sig:
|
||||||
|
sigs[(r['cfg'], verb, sig[:70])].append(r['eval'][:30])
|
||||||
|
for cmd, sig in r['other_fail']:
|
||||||
|
sigs[(r['cfg'], 'bash:'+cmd[:40], sig[:70])].append(r['eval'][:30])
|
||||||
|
for (cfg, verb, sig), evals in sorted(sigs.items()):
|
||||||
|
print(f"[{cfg:12}] {verb:32} n={len(evals):2} {sig}")
|
||||||
|
# ---- 3. first-attempt success per verb, per config ----
|
||||||
|
print()
|
||||||
|
print('=' * 110)
|
||||||
|
print('ATTEMPTS-TO-SUCCESS PER CLI VERB (per config)')
|
||||||
|
print('=' * 110)
|
||||||
|
agg = defaultdict(lambda: defaultdict(lambda: {'sess':0,'first':0,'never':0,'retry_hist':[]}))
|
||||||
|
for r in rows:
|
||||||
|
st = attempts_to_success(r['cli'])
|
||||||
|
for verb, s in st.items():
|
||||||
|
a = agg[r['cfg']][verb]
|
||||||
|
a['sess'] += s['total_sessions']; a['first'] += s['first_try']; a['never'] += s['never']
|
||||||
|
a['retry_hist'] += [x for x in s['fails_before'] if x]
|
||||||
|
for cfg in agg:
|
||||||
|
print(f'--- {cfg} ---')
|
||||||
|
for verb, a in sorted(agg[cfg].items(), key=lambda kv: -kv[1]['sess']):
|
||||||
|
ft = f"{a['first']}/{a['sess']}" if a['sess'] else '-'
|
||||||
|
print(f" {verb:34} sessions={a['sess']:3} first-try={ft:7} never-ok={a['never']:2} retry-dist={sorted(set(a['retry_hist'])) if a['retry_hist'] else []}")
|
||||||
|
# ---- 4. failed runs: what happened ----
|
||||||
|
print()
|
||||||
|
print('=' * 110)
|
||||||
|
print('FAILED/TIMEOUT RUNS — trajectory tail')
|
||||||
|
print('=' * 110)
|
||||||
|
for r in rows:
|
||||||
|
if r['passed'] is False or not r['settled']:
|
||||||
|
fails = [(v, s) for v, c, s in r['cli'] if s] + [(c[:50], s) for c, s in r['other_fail']]
|
||||||
|
print(f"\n{r['ws']} {r['iter']}/{r['eval']}/{r['cfg']} passed={r['passed']} settled={r['settled']}")
|
||||||
|
for v, s in fails[-6:]:
|
||||||
|
print(f" FAIL {v}: {s}")
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,376 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Grade iteration-4 A/B rerun evals.
|
||||||
|
|
||||||
|
Same checks as the original grade_evals.py, adapted to per-config sandboxes:
|
||||||
|
- each config (with_skill/old_skill) has its own base + order_logs table + ORD record
|
||||||
|
- CLI = absolute path to the NEW CLI (reads only; version-agnostic)
|
||||||
|
Usage: python3 grade_evals4.py <iteration-name> <tag>
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
HARNESS = Path(__file__).parent # evals/ itself
|
||||||
|
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
|
||||||
|
CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable")
|
||||||
|
env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'sandbox-env.json'))).read_text())
|
||||||
|
|
||||||
|
|
||||||
|
def cli_json(cfg, *args):
|
||||||
|
e = {**os.environ,
|
||||||
|
"INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
|
||||||
|
"INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
|
||||||
|
"INKREACH_TEABLE_BASE_ID": cfg["INKREACH_TEABLE_BASE_ID"]}
|
||||||
|
p = subprocess.run([CLI, *args, "--json"], capture_output=True, text=True, env=e, timeout=60)
|
||||||
|
try:
|
||||||
|
return json.loads(p.stdout)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def transcript(run_dir: Path):
|
||||||
|
cmds, reads = [], []
|
||||||
|
for line in (run_dir / "pi.jsonl").read_text(errors="replace").splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
ev = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if ev.get("type") != "tool_execution_start":
|
||||||
|
continue
|
||||||
|
if ev.get("toolName") == "bash":
|
||||||
|
cmds.append(ev.get("args", {}).get("command", ""))
|
||||||
|
elif ev.get("toolName") == "read":
|
||||||
|
reads.append(ev.get("args", {}).get("path", ""))
|
||||||
|
return cmds, reads
|
||||||
|
|
||||||
|
|
||||||
|
def answer(run_dir: Path) -> str:
|
||||||
|
fa = run_dir / "final_answer.md"
|
||||||
|
af = run_dir / "assistant_final.md"
|
||||||
|
return (fa.read_text(errors="replace") if fa.exists() else af.read_text(errors="replace"))
|
||||||
|
|
||||||
|
|
||||||
|
def used_official_teable(cmds):
|
||||||
|
for c in cmds:
|
||||||
|
# probing for existence (command -v / which) is prudent, not usage
|
||||||
|
probe = re.search(r"(command -v|which)\s+teable", c)
|
||||||
|
if probe:
|
||||||
|
continue
|
||||||
|
if re.search(r"(^|[^a-z-])teable\s", c) or "npx teable" in c or re.search(r"npm\s+(i|install)\s+teable\b", c):
|
||||||
|
return c
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def find_wf(cfg, name_part):
|
||||||
|
wfs = cli_json(cfg, "workflow", "list") or []
|
||||||
|
if not isinstance(wfs, list):
|
||||||
|
return None # error payload ({"error":...}) — not a workflow list
|
||||||
|
for w in wfs:
|
||||||
|
if isinstance(w, dict) and name_part in w.get("name", ""):
|
||||||
|
return w
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def wf_nodes(cfg, wf_id):
|
||||||
|
wf = cli_json(cfg, "workflow", "get", wf_id) or {}
|
||||||
|
return wf, [n for n in wf.get("nodes", []) if n.get("type") == "trigger"], [n for n in wf.get("nodes", []) if n.get("type") == "action"]
|
||||||
|
|
||||||
|
|
||||||
|
def result(text, passed, evidence):
|
||||||
|
return {"text": text, "passed": bool(passed), "evidence": str(evidence)[:400]}
|
||||||
|
|
||||||
|
|
||||||
|
def grade_eval(cfg, ev_name, tag, run_dir, cfg_short="N"):
|
||||||
|
tbl = cfg["TBL5_ID"]
|
||||||
|
ord_id = cfg["ORD"]
|
||||||
|
cmds, reads = transcript(run_dir)
|
||||||
|
ans = answer(run_dir)
|
||||||
|
all_cmds = "\n".join(cmds)
|
||||||
|
out = []
|
||||||
|
|
||||||
|
if ev_name == "button-click-automation":
|
||||||
|
# Locate the workflow via the button-field binding (options.workflow) —
|
||||||
|
# the prompt only fixes the FIELD name, agents may name the workflow
|
||||||
|
# anything (e.g. 催单自动化A4W), so searching by workflow name misfires.
|
||||||
|
fields = cli_json(cfg, "field", "list", tbl) or []
|
||||||
|
btn = next((f for f in fields if f.get("name") == f"催单按钮{tag}"), None)
|
||||||
|
wf = None
|
||||||
|
if btn:
|
||||||
|
wf_id = ((btn.get("options") or {}).get("workflow") or {}).get("id")
|
||||||
|
if wf_id:
|
||||||
|
wf = cli_json(cfg, "workflow", "get", wf_id)
|
||||||
|
if not wf:
|
||||||
|
wf = find_wf(cfg, f"催单按钮{tag}")
|
||||||
|
if not wf:
|
||||||
|
out.append(result("按钮字段绑定的工作流存在 (催单按钮%s)" % tag, False,
|
||||||
|
"field list 无该按钮字段或其 options.workflow 无效"))
|
||||||
|
return out
|
||||||
|
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
|
||||||
|
out.append(result("工作流已激活 (isActive)", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
|
||||||
|
fields = cli_json(cfg, "field", "list", tbl) or []
|
||||||
|
btn = next((f for f in fields if f.get("name") == f"催单按钮{tag}"), None)
|
||||||
|
if not btn:
|
||||||
|
out.append(result("创建按钮字段 催单按钮%s" % tag, False, "field list 未找到该字段"))
|
||||||
|
else:
|
||||||
|
wf_opt = (btn.get("options") or {}).get("workflow")
|
||||||
|
ok = isinstance(wf_opt, dict) and wf_opt.get("id") == wf["id"]
|
||||||
|
out.append(result("按钮字段 options.workflow 为对象且指向该工作流", ok,
|
||||||
|
"options.workflow=%r" % wf_opt))
|
||||||
|
out.append(result("触发器为 buttonClick", any(t.get("category") == "buttonClick" for t in triggers),
|
||||||
|
"trigger categories=%s" % [t.get("category") for t in triggers]))
|
||||||
|
script = next((a for a in actions if a.get("category") == "script"), None)
|
||||||
|
out.append(result("script action 读取 triggerInput", script and "triggerInput" in (script.get("config") or {}).get("code", ""),
|
||||||
|
"code=%r" % ((script or {}).get("config") or {}).get("code", "")[:120]))
|
||||||
|
fired = any("fire button-click" in c for c in cmds)
|
||||||
|
runs_any = cli_json(cfg, "runs", "list", wf["id"]) or []
|
||||||
|
verified = any(re.search(r"runs (list|get)", c) for c in cmds) or len(runs_any) > 0
|
||||||
|
out.append(result("用 fire button-click 实测", fired, [c for c in cmds if "fire" in c][:2]))
|
||||||
|
out.append(result("执行后查看 runs 结果", verified, [c for c in cmds if "runs" in c][:2]))
|
||||||
|
runs = cli_json(cfg, "runs", "list", wf["id"]) or []
|
||||||
|
ok_run = any(r.get("status") in ("succeeded", "success") for r in runs)
|
||||||
|
out.append(result("存在 succeeded 的执行记录", ok_run, "statuses=%s" % [r.get("status") for r in runs][:5]))
|
||||||
|
bad = used_official_teable(cmds)
|
||||||
|
out.append(result("全程使用 inkreach-teable CLI(未用官方 teable)", bad is None, bad or "ok"))
|
||||||
|
|
||||||
|
elif ev_name == "scheduled-digest":
|
||||||
|
wf = find_wf(cfg, f"每日统计{tag}")
|
||||||
|
if not wf:
|
||||||
|
out.append(result("创建名为 每日统计%s 的工作流" % tag, False, "workflow list 中未找到"))
|
||||||
|
return out
|
||||||
|
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
|
||||||
|
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
|
||||||
|
trig = triggers[0] if triggers else {}
|
||||||
|
timing = (trig.get("config") or {}).get("timing") or {}
|
||||||
|
ignored = [k for k in ("triggerTime", "weekdays", "daysOfMonth", "starting", "ending", "timezone", "tz") if k in timing]
|
||||||
|
out.append(result("scheduledTime 触发器", trig.get("category") == "scheduledTime", "category=%s" % trig.get("category")))
|
||||||
|
out.append(result("timing 只含生效字段(type+interval,无被忽略字段)", trig.get("category") != "scheduledTime" or (not ignored and "type" in timing and "interval" in timing),
|
||||||
|
"timing=%r ignored=%s" % (timing, ignored)))
|
||||||
|
has_stats = any(a.get("category") == "script" and re.search(r"record|count|统计|length", (a.get("config") or {}).get("code", "")) for a in actions) or any(
|
||||||
|
a.get("category") in ("getRecords",) for a in actions)
|
||||||
|
out.append(result("有统计逻辑的 action(script 读记录数或 getRecords)", has_stats,
|
||||||
|
"action categories=%s" % [a.get("category") for a in actions]))
|
||||||
|
|
||||||
|
elif ev_name == "record-condition-email":
|
||||||
|
wf = find_wf(cfg, f"失败告警{tag}")
|
||||||
|
if not wf:
|
||||||
|
out.append(result("创建名为 失败告警%s 的工作流" % tag, False, "workflow list 中未找到"))
|
||||||
|
return out
|
||||||
|
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
|
||||||
|
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
|
||||||
|
trig = triggers[0] if triggers else {}
|
||||||
|
out.append(result("触发器 category 为 recordCondition", trig.get("category") == "recordCondition",
|
||||||
|
"category=%s" % trig.get("category")))
|
||||||
|
c = trig.get("config") or {}
|
||||||
|
cond_ok = "conditions" in c and "filter" not in c
|
||||||
|
out.append(result("条件键名为 conditions(无 filter)", cond_ok, "config keys=%s" % list(c.keys())))
|
||||||
|
if isinstance(c.get("conditions"), dict):
|
||||||
|
out.append(result("conditions 形如 {conjunction, filterSet}", "filterSet" in c["conditions"],
|
||||||
|
"conditions keys=%s" % list(c["conditions"].keys())))
|
||||||
|
bad_key = "recordMatchesConditions" in all_cmds
|
||||||
|
out.append(result("未使用官方键名 recordMatchesConditions", not bad_key, "transcript" if bad_key else "ok"))
|
||||||
|
email = next((a for a in actions if a.get("category") in ("sendEmail", "email")), None)
|
||||||
|
ok_email = email and "ops@xingchen.tech" in json.dumps(email.get("config") or {}, ensure_ascii=False)
|
||||||
|
out.append(result("sendEmail action 收件人 ops@xingchen.tech", ok_email,
|
||||||
|
"email action config=%s" % json.dumps((email or {}).get("config") or {}, ensure_ascii=False)[:200]))
|
||||||
|
|
||||||
|
elif ev_name == "debug-failed-run":
|
||||||
|
out.append(result("查询了工作流状态 (workflow get/list)", any(re.search(r"workflow (get|list)", c) for c in cmds),
|
||||||
|
[c for c in cmds if "workflow" in c][:2]))
|
||||||
|
out.append(result("查询失败执行 (runs list --status failed 或过滤)", any(re.search(r"runs list", c) for c in cmds),
|
||||||
|
[c for c in cmds if "runs" in c][:3]))
|
||||||
|
detail_in_ans = bool(re.search(r"boom|nodeResults|错误详情|执行失败原因|throw new Error", ans))
|
||||||
|
out.append(result("查看了具体执行详情 (runs get 或答复引用运行错误详情)",
|
||||||
|
any(re.search(r"runs get", c) for c in cmds) or detail_in_ans,
|
||||||
|
[c for c in cmds if "runs get" in c][:2] or ans[:120]))
|
||||||
|
diagnosed = re.search(r"boom|timeout|超时|抛错|throw", ans, re.I)
|
||||||
|
out.append(result("最终答复指出脚本抛错原因", bool(diagnosed), ans[:200]))
|
||||||
|
reactivated = [c for c in cmds if re.search(r"workflow activate", c)]
|
||||||
|
out.append(result("未盲目重新激活(诊断前不改)", not reactivated, reactivated or "ok"))
|
||||||
|
|
||||||
|
elif ev_name == "select-constraint-write":
|
||||||
|
inspected = any(re.search(r"field (list|get|visible-choices)", c) for c in cmds)
|
||||||
|
out.append(result("写入前检查了字段约束 (field list --with-options / field get)", inspected,
|
||||||
|
[c for c in cmds if "field" in c][:2]))
|
||||||
|
fields = cli_json(cfg, "field", "list", tbl) or []
|
||||||
|
f = next((x for x in fields if x.get("name") == "结果"), {})
|
||||||
|
choices = [c.get("name") if isinstance(c, dict) else c for c in ((f.get("options") or {}).get("choices") or [])]
|
||||||
|
added = "已完成" in choices
|
||||||
|
recs = (cli_json(cfg, "record", "get", "--table-id", tbl) or {}).get("records") or []
|
||||||
|
rec = next((r for r in recs if (r.get("fields") or {}).get("订单号") == ord_id), {})
|
||||||
|
val = (rec.get("fields") or {}).get("结果")
|
||||||
|
claims_success = bool(re.search(r"(✅|已.{0,3}(改|更新|设置|写入).{0,20}已完成|成功.{0,10}改为)", ans))
|
||||||
|
if added and val == "已完成":
|
||||||
|
out.append(result("合法路径:先加选项再写入,记录值=已完成", True, "choices=%s val=%s" % (choices, val)))
|
||||||
|
elif claims_success and val != "已完成" and not added:
|
||||||
|
out.append(result("未发生静默失败(声称成功但值未变)", False,
|
||||||
|
"答复声称已改为「已完成」,但实际 choices=%s 记录值=%r —— 教科书式 mutationApplied=false 静默失败" % (choices, val)))
|
||||||
|
elif val in (None, "处理中") and re.search(r"不在|无法|无效|不是.{0,6}选项|未写入|没有写入|未添加|无法直接|不能直接", ans):
|
||||||
|
out.append(result("识别出「已完成」不是有效选项并向用户说明(未静默失败)", True, ans[:250]))
|
||||||
|
else:
|
||||||
|
out.append(result("写入了有效选项值或正确说明", val == "已完成",
|
||||||
|
"choices=%s val=%r claims_success=%s ans=%s" % (choices, val, claims_success, ans[:150])))
|
||||||
|
listed = all(x in ans for x in ("成功", "失败", "处理中"))
|
||||||
|
out.append(result("答复列出了全部可选值(成功/失败/处理中)", listed, ans[:200]))
|
||||||
|
|
||||||
|
elif ev_name == "official-cli-near-miss":
|
||||||
|
bad = used_official_teable(cmds)
|
||||||
|
out.append(result("未安装/调用官方 teable CLI 打实例", bad is None, bad or "ok"))
|
||||||
|
out.append(result("使用了 inkreach-teable CLI", any("inkreach-teable" in c for c in cmds),
|
||||||
|
[c for c in cmds if "inkreach-teable" in c][:2]))
|
||||||
|
out.append(result("答复列出了 order_logs 表", "order_logs" in ans, ans[:200]))
|
||||||
|
|
||||||
|
elif ev_name == "webhook-receiver":
|
||||||
|
wf = find_wf(cfg, f"回单通知{tag}")
|
||||||
|
if not wf:
|
||||||
|
out.append(result("创建名为 回单通知%s 的工作流" % tag, False, "workflow list 中未找到"))
|
||||||
|
return out
|
||||||
|
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
|
||||||
|
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
|
||||||
|
trig = triggers[0] if triggers else {}
|
||||||
|
out.append(result("触发器为 webhook", trig.get("category") == "webhook", "category=%s" % trig.get("category")))
|
||||||
|
base = cfg["INKREACH_TEABLE_BASE_ID"]
|
||||||
|
correct = f"/api/webhook/{base}/{wf['id']}"
|
||||||
|
out.append(result("答复给出正确 URL 形状 /api/webhook/{baseId}/{workflowId}(无 /base/ 段)", correct in ans,
|
||||||
|
"期望含 %s;答复=%s" % (correct, ans[:180])))
|
||||||
|
tested = any(re.search(r"fire webhook", c) for c in cmds) or any("api/webhook" in c and re.search(r"curl|POST", c) for c in cmds)
|
||||||
|
out.append(result("实际测试了 webhook(fire webhook 或 curl POST)", tested,
|
||||||
|
[c for c in cmds if "webhook" in c][:3]))
|
||||||
|
|
||||||
|
elif ev_name == "weekday-schedule-trap":
|
||||||
|
wf = find_wf(cfg, f"工作日盘点{tag}")
|
||||||
|
if not wf:
|
||||||
|
out.append(result("创建名为 工作日盘点%s 的工作流" % tag, False, "workflow list 中未找到"))
|
||||||
|
return out
|
||||||
|
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
|
||||||
|
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
|
||||||
|
trig = triggers[0] if triggers else {}
|
||||||
|
timing = (trig.get("config") or {}).get("timing") or {}
|
||||||
|
ignored = [k for k in ("triggerTime", "weekdays", "daysOfMonth", "starting", "ending", "timezone", "tz") if k in timing]
|
||||||
|
out.append(result("scheduledTime 触发器且 timing 只含生效字段", trig.get("category") == "scheduledTime" and not ignored and "type" in timing,
|
||||||
|
"category=%s timing=%r ignored=%s" % (trig.get("category"), timing, ignored)))
|
||||||
|
warns = re.search(r"(忽略|不支持|仅|只).{0,30}(interval|间隔|type)|weekday|工作日.{0,10}(不|无法|忽略)|backend ignores", ans, re.I)
|
||||||
|
out.append(result("答复如实告知『具体时刻/星期』不被后端支持", bool(warns), ans[:250]))
|
||||||
|
claims_exact = re.search(r"(已.{0,6}(配好|设置).{0,40}(工作日|下午\s*5|17:00))|每个工作日下午 ?5 ?点.{0,10}(会|将)跑", ans)
|
||||||
|
out.append(result("未虚假承诺『工作日下午5点会照常执行』", not claims_exact, ans[:200]))
|
||||||
|
|
||||||
|
elif ev_name == "secret-env-usage":
|
||||||
|
wf = find_wf(cfg, f"汇率查询{tag}")
|
||||||
|
if not wf:
|
||||||
|
out.append(result("创建名为 汇率查询%s 的工作流" % tag, False, "workflow list 中未找到"))
|
||||||
|
return out
|
||||||
|
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
|
||||||
|
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
|
||||||
|
secrets = cli_json(cfg, "secret", "list", wf["id"]) or []
|
||||||
|
has_secret = bool(secrets)
|
||||||
|
out.append(result("存了密钥(非明文入代码;secret set 或 secret list 可见)",
|
||||||
|
any(re.search(r"secret set", c) for c in cmds) or has_secret,
|
||||||
|
[c for c in cmds if "secret set" in c][:1] or f"secret list: {len(secrets)} 条"))
|
||||||
|
script = next((a for a in actions if a.get("category") == "script"), None)
|
||||||
|
code = (script or {}).get("config", {}).get("code", "")
|
||||||
|
out.append(result("脚本经 process.env 读取密钥", bool(re.search(r"process\.env\.[A-Z_]+", code)), code[:150]))
|
||||||
|
out.append(result("脚本/工作流配置中无明文 key(sk-test-abc123)", "sk-test-abc123" not in json.dumps(wf_full, ensure_ascii=False),
|
||||||
|
"明文检查通过" if "sk-test-abc123" not in json.dumps(wf_full, ensure_ascii=False) else "配置含明文!"))
|
||||||
|
|
||||||
|
elif ev_name == "script-npm-deps":
|
||||||
|
wf = find_wf(cfg, f"日期演示{tag}")
|
||||||
|
if not wf:
|
||||||
|
out.append(result("创建名为 日期演示%s 的工作流" % tag, False, "workflow list 中未找到"))
|
||||||
|
return out
|
||||||
|
wf_full, triggers, actions = wf_nodes(cfg, wf["id"])
|
||||||
|
out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive")))
|
||||||
|
script = next((a for a in actions if a.get("category") == "script"), None)
|
||||||
|
code = (script or {}).get("config", {}).get("code", "") if script else ""
|
||||||
|
out.append(result("脚本使用 dayjs", "dayjs" in code, code[:150]))
|
||||||
|
out.append(result("依赖经 --dependencies 声明(配置含 dayjs)", "dayjs" in json.dumps((script or {}).get("config", {})),
|
||||||
|
json.dumps((script or {}).get("config", {}), ensure_ascii=False)[:200]))
|
||||||
|
out.append(result("知晓后端 npm PATH 前提(答复或命令体现)", any(re.search(r"--dependencies", c) for c in cmds) or bool(re.search(r"PATH|npm|后端|service", ans)),
|
||||||
|
ans[:180]))
|
||||||
|
|
||||||
|
elif ev_name == "lowperm-403-handling":
|
||||||
|
ans_l = ans.lower()
|
||||||
|
perm_ack = bool(re.search(r"权限|403|无权|forbidden|没.{0,4}(资格|权限)|令牌|token.{0,12}(无|不|失效)|授权", ans, re.I))
|
||||||
|
fake_success = bool(re.search(r"(已(经)?(建|创建|完成)[了成]?.{0,10}(表|快递)|(表|快递).{0,8}(已|成功)(建|创建)|成功创建.{0,6}表)", ans))
|
||||||
|
cli_tried = any("inkreach-teable" in c for c in cmds)
|
||||||
|
out.append(result("尝试用 CLI 执行任务", cli_tried, [c for c in cmds if "inkreach-teable" in c][:1]))
|
||||||
|
out.append(result("识别出权限不足并如实说明", perm_ack and not fake_success, ans[:220]))
|
||||||
|
# 未陷入 403 循环(超过 8 次失败重试视为 flailing)
|
||||||
|
f403 = sum(1 for c in cmds if re.search(r"inkreach-teable", c))
|
||||||
|
out.append(result("未陷入无意义重试(命令次数有界)", f403 <= 12, f"cli calls={f403}"))
|
||||||
|
|
||||||
|
elif ev_name == "deep-pagination":
|
||||||
|
tbl13 = cfg.get("TBL13_ID")
|
||||||
|
if not tbl13:
|
||||||
|
out.append(result("fixture big_logs 存在", False, "TBL13_ID 未配置"))
|
||||||
|
return out
|
||||||
|
recs = (cli_json(cfg, "record", "get", "--table-id", tbl13, "--take", "1000") or {}).get("records") or []
|
||||||
|
slow = [r for r in recs if str((r.get("fields") or {}).get("订单号", "")).startswith("SLOW-")]
|
||||||
|
ids = [str((r.get("fields") or {}).get("订单号")) for r in slow]
|
||||||
|
count_ok = bool(re.search(r"2\s*(条|个|records?|行)", ans))
|
||||||
|
found_any = any(i in ans for i in ids)
|
||||||
|
out.append(result("正确给出 SLOW- 记录总数=2", count_ok, f"实际: {ids}"))
|
||||||
|
out.append(result("答复包含一条真实 SLOW- 记录的订单号", found_any, ans[:200]))
|
||||||
|
|
||||||
|
elif ev_name == "vague-daily-notify":
|
||||||
|
# 模糊需求:每天 + 订单进展 + 站内通知 → scheduledTime + 统计 + sendNotification + active
|
||||||
|
wfs = cli_json(cfg, "workflow", "list") or []
|
||||||
|
cands = [w for w in wfs if w.get("isActive")]
|
||||||
|
hit = None
|
||||||
|
for w in cands:
|
||||||
|
wf_full, trs, acs = wf_nodes(cfg, w["id"])
|
||||||
|
if (trs and trs[0].get("category") == "scheduledTime"
|
||||||
|
and any(a.get("category") == "sendNotification" for a in acs)):
|
||||||
|
hit = wf_full
|
||||||
|
break
|
||||||
|
out.append(result("建了激活的每日定时工作流(scheduledTime+sendNotification)", hit is not None,
|
||||||
|
f"active scheduledTime+notify wf: {hit.get('name') if hit else None}"))
|
||||||
|
if hit:
|
||||||
|
_, trs, acs = wf_nodes(cfg, hit["id"])
|
||||||
|
trig = trs[0] if trs else {}
|
||||||
|
timing = (trig.get("config") or {}).get("timing") or {}
|
||||||
|
out.append(result("timing 为生效字段(type+interval)", "type" in timing and "interval" in timing, f"timing={timing}"))
|
||||||
|
has_stats = any(a.get("category") == "script" and re.search(r"record|count|统计|length", (a.get("config") or {}).get("code", "")) for a in acs)
|
||||||
|
out.append(result("有订单统计逻辑", has_stats, [a.get("category") for a in acs]))
|
||||||
|
|
||||||
|
elif ev_name == "field-options-update":
|
||||||
|
fields = cli_json(cfg, "field", "list", tbl) or []
|
||||||
|
f = next((x for x in fields if x.get("name") == "结果"), {})
|
||||||
|
choices = [c.get("name") if isinstance(c, dict) else c for c in ((f.get("options") or {}).get("choices") or [])]
|
||||||
|
out.append(result("「结果」字段 choices 新增「已取消」", "已取消" in choices, "choices=%s" % choices))
|
||||||
|
out.append(result("使用 field update --options(走 convert API)", any(re.search(r"field update.*--options|field.*--options", c, re.S) for c in cmds),
|
||||||
|
[c for c in cmds if "field" in c and "update" in c][:1]))
|
||||||
|
out.append(result("原有选项未被破坏(成功/失败/处理中 仍在)", all(x in choices for x in ("成功", "失败", "处理中")), "choices=%s" % choices))
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
iteration = sys.argv[1]
|
||||||
|
tag = sys.argv[2]
|
||||||
|
iteration_dir = ROOT / iteration
|
||||||
|
for eval_dir in sorted(iteration_dir.glob("eval-*")):
|
||||||
|
ev_name = eval_dir.name.split("-", 2)[2]
|
||||||
|
for config_dir in sorted(p for p in eval_dir.iterdir() if p.is_dir()):
|
||||||
|
run_dir = config_dir / "run-1"
|
||||||
|
if not (run_dir / "pi.jsonl").exists():
|
||||||
|
continue
|
||||||
|
cfg = env_all.get(config_dir.name) or env_all['with_skill']
|
||||||
|
cfg_short = "W" if config_dir.name == "with_skill" else "O"
|
||||||
|
tag_cfg = f"{tag}{cfg_short}"
|
||||||
|
try:
|
||||||
|
expectations = grade_eval(cfg, ev_name, tag_cfg, run_dir, cfg_short)
|
||||||
|
except Exception as e:
|
||||||
|
expectations = [result("判分异常", False, repr(e))]
|
||||||
|
g = {"expectations": expectations}
|
||||||
|
(run_dir / "grading.json").write_text(json.dumps(g, indent=2, ensure_ascii=False))
|
||||||
|
npass = sum(1 for x in expectations if x["passed"])
|
||||||
|
print(f"{eval_dir.name:38} {config_dir.name:12} {npass}/{len(expectations)}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Grade multi-turn (correction) evals: final state must satisfy the CORRECTED
|
||||||
|
requirement; amendment of the existing artifact is preferred over rebuilding.
|
||||||
|
|
||||||
|
mt-1 accepted shapes for "结果 变成 失败 才站内通知":
|
||||||
|
A) recordCondition trigger with `conditions` (state-match reading), OR
|
||||||
|
B) recordUpdated trigger watching the 结果 field + script gate on 失败
|
||||||
|
(transition reading — semantically at least as correct)
|
||||||
|
mt-2: incremental schema (add 渠道 select + mark the existing record).
|
||||||
|
"""
|
||||||
|
import json, os, re, subprocess, sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
HARNESS = Path(__file__).parent
|
||||||
|
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
|
||||||
|
env_all = json.loads(Path(os.environ.get("EVAL_ENV", str(HARNESS.parent / "sandbox-env.json"))).read_text())
|
||||||
|
CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable")
|
||||||
|
|
||||||
|
|
||||||
|
def cli(*a):
|
||||||
|
env = {**os.environ,
|
||||||
|
"INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
|
||||||
|
"INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
|
||||||
|
"INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"]}
|
||||||
|
p = subprocess.run([CLI, *a, "--json"], capture_output=True, text=True, env=env, timeout=60)
|
||||||
|
try:
|
||||||
|
return json.loads(p.stdout)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def result(t, p, e):
|
||||||
|
return {"text": t, "passed": bool(p), "evidence": str(e)[:300]}
|
||||||
|
|
||||||
|
|
||||||
|
def cmds_of(rd, turn=2):
|
||||||
|
out = []
|
||||||
|
f = rd / f"turn{turn}.jsonl"
|
||||||
|
if not f.exists():
|
||||||
|
return out
|
||||||
|
for line in f.read_text(errors="replace").splitlines():
|
||||||
|
try:
|
||||||
|
e = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if e.get("type") == "tool_execution_start" and e.get("toolName") == "bash":
|
||||||
|
out.append(e.get("args", {}).get("command", ""))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def grade_mt1():
|
||||||
|
out = []
|
||||||
|
wfs = cli("workflow", "list") or []
|
||||||
|
if not isinstance(wfs, list):
|
||||||
|
return [result("workflow list 可读", False, wfs)]
|
||||||
|
ok = amended = False
|
||||||
|
for w in wfs:
|
||||||
|
full = cli("workflow", "get", w["id"]) or {}
|
||||||
|
trs = [n for n in full.get("nodes", []) if n.get("type") == "trigger"]
|
||||||
|
acs = [n for n in full.get("nodes", []) if n.get("type") == "action"]
|
||||||
|
trig = trs[0] if trs else {}
|
||||||
|
cfg = trig.get("config") or {}
|
||||||
|
cats = [a.get("category") for a in acs]
|
||||||
|
shape_a = trig.get("category") == "recordCondition" and "conditions" in cfg
|
||||||
|
shape_b = (trig.get("category") == "recordUpdated" and cfg.get("fieldIds")
|
||||||
|
and any(a.get("category") == "script" and re.search(r"失败", (a.get("config") or {}).get("code", "")) for a in acs))
|
||||||
|
if full.get("isActive") and (shape_a or shape_b):
|
||||||
|
ok = "sendNotification" in cats and "sendEmail" not in cats
|
||||||
|
out.append(result("修正后终态:仅在结果变失败时站内通知(recordCondition 或 recordUpdated+门控)", ok,
|
||||||
|
[w.get("name") for w in wfs]))
|
||||||
|
c2 = cmds_of(ROOT / "iteration-mt" / "mt-1-notify-correction" / "auto_skill" / "run-1")
|
||||||
|
amended = any(re.search(r"trigger set|action (update|add|delete)", c) for c in c2) and not any(
|
||||||
|
re.search(r"workflow create", c) for c in c2)
|
||||||
|
out.append(result("第二轮是修改现有工作流而非推倒重建", amended,
|
||||||
|
[c[:80] for c in c2 if "workflow" in c][:3]))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def grade_mt2():
|
||||||
|
out = []
|
||||||
|
tables = cli("table", "list") or []
|
||||||
|
fb = [t for t in tables if "反馈" in t.get("name", "") or "feedback" in t.get("name", "").lower()]
|
||||||
|
if not fb:
|
||||||
|
return [result("反馈表存在", False, [t.get("name") for t in tables])]
|
||||||
|
tid = fb[0]["id"]
|
||||||
|
flds = cli("field", "list", tid) or []
|
||||||
|
okf = any(f.get("name") == "内容" for f in flds)
|
||||||
|
ch = next((f for f in flds if f.get("name") == "渠道"), None)
|
||||||
|
okc = bool(ch) and set(["微信", "电话", "邮件"]) <= set(
|
||||||
|
c.get("name") for c in (ch.get("options") or {}).get("choices", []))
|
||||||
|
recs = (cli("record", "get", "--table-id", tid) or {}).get("records") or []
|
||||||
|
okr = any((r.get("fields") or {}).get("渠道") == "微信" for r in recs)
|
||||||
|
out.append(result("第一轮产物保留(表+内容字段)", okf, fb[0]["name"]))
|
||||||
|
out.append(result("第二轮增量:渠道字段三选项", okc, (ch or {}).get("options")))
|
||||||
|
out.append(result("示例记录渠道=微信", okr, [(r.get("fields") or {}).get("订单号"), (r.get("fields") or {}).get("渠道")] if not okr else "ok"))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
iteration = sys.argv[1] if len(sys.argv) > 1 else "iteration-mt"
|
||||||
|
g1 = grade_mt1()
|
||||||
|
g2 = grade_mt2()
|
||||||
|
for name, exps in [("mt-1-notify-correction", g1), ("mt-2-incremental-schema", g2)]:
|
||||||
|
rd = ROOT / iteration / name / "auto_skill" / "run-1"
|
||||||
|
(rd / "grading.json").write_text(json.dumps({"expectations": exps}, indent=2, ensure_ascii=False))
|
||||||
|
np = sum(1 for x in exps if x["passed"])
|
||||||
|
print(f"{name:28} {np}/{len(exps)}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Grade plugin-skill eval runs (ported from plugin-workspace, per-config sandboxes)."""
|
||||||
|
import json, os, re, subprocess, sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
HARNESS = Path(__file__).parent # evals/ itself
|
||||||
|
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
|
||||||
|
CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable")
|
||||||
|
env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'plugin-sandbox-env.json'))).read_text())
|
||||||
|
|
||||||
|
FORBIDDEN_IMPORT = re.compile(r"^\s*import\s+(?!type)[^\n]*from\s+['\"]@teable/(sdk|ui-lib|openapi|core|icons)", re.M)
|
||||||
|
REACT_IMPORT = re.compile(r"^\s*import\s+(?!type)[^\n]*from\s+['\"]react['\"]", re.M)
|
||||||
|
|
||||||
|
|
||||||
|
def cli_json(*args):
|
||||||
|
e = {**os.environ,
|
||||||
|
"INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
|
||||||
|
"INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
|
||||||
|
"INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"]}
|
||||||
|
p = subprocess.run([CLI, *args, "--json"], capture_output=True, text=True, env=e, timeout=60)
|
||||||
|
try:
|
||||||
|
return json.loads(p.stdout)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def transcript(run_dir: Path):
|
||||||
|
cmds = []
|
||||||
|
for line in (run_dir / "pi.jsonl").read_text(errors="replace").splitlines():
|
||||||
|
try:
|
||||||
|
ev = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if ev.get("type") == "tool_execution_start" and ev.get("toolName") == "bash":
|
||||||
|
cmds.append(ev.get("args", {}).get("command", ""))
|
||||||
|
return cmds
|
||||||
|
|
||||||
|
|
||||||
|
def answer(run_dir: Path) -> str:
|
||||||
|
fa, af = run_dir / "final_answer.md", run_dir / "assistant_final.md"
|
||||||
|
return fa.read_text(errors="replace") if fa.exists() else af.read_text(errors="replace")
|
||||||
|
|
||||||
|
|
||||||
|
def plugin_sources(run_dir: Path):
|
||||||
|
files = sorted((run_dir / "plugin").rglob("*.[tj]s*")) or sorted(run_dir.rglob("plugin.json"))
|
||||||
|
return "\n".join(p.read_text(errors="replace") for p in files), files
|
||||||
|
|
||||||
|
|
||||||
|
def result(text, passed, evidence):
|
||||||
|
return {"text": text, "passed": bool(passed), "evidence": str(evidence)[:400]}
|
||||||
|
|
||||||
|
|
||||||
|
def grade_eval(ev_name, run_dir: Path, cfg_short: str):
|
||||||
|
cmds = transcript(run_dir)
|
||||||
|
ans = answer(run_dir)
|
||||||
|
out = []
|
||||||
|
|
||||||
|
if ev_name in ("contextmenu-plugin", "panel-widget"):
|
||||||
|
src, files = plugin_sources(run_dir)
|
||||||
|
want_pos = "contextMenu" if ev_name == "contextmenu-plugin" else "panel"
|
||||||
|
want_api = "addContextMenuAction" if ev_name == "contextmenu-plugin" else "addPanel"
|
||||||
|
pj = run_dir / "plugin" / "plugin.json"
|
||||||
|
manifest_ok = False
|
||||||
|
if pj.exists():
|
||||||
|
try:
|
||||||
|
m = json.loads(pj.read_text())
|
||||||
|
manifest_ok = bool(re.fullmatch(r"[a-z0-9-]+", m.get("id", ""))) and want_pos in (m.get("positions") or [])
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
out.append(result(f"plugin.json 存在且 manifest 合法(id 合规、positions 含 {want_pos})", manifest_ok,
|
||||||
|
pj.read_text(errors="replace")[:150] if pj.exists() else "plugin/plugin.json 未找到"))
|
||||||
|
out.append(result("代码从 window.__TEABLE_HOST__ 获取宿主能力", "__TEABLE_HOST__" in src,
|
||||||
|
"host global: %s" % ("有" if "__TEABLE_HOST__" in src else "无")))
|
||||||
|
bad = FORBIDDEN_IMPORT.findall(src)
|
||||||
|
out.append(result("@teable/* 无运行时值导入(仅允许 import type)", not bad, "违规: %r" % bad if bad else "ok"))
|
||||||
|
uses_react = bool(REACT_IMPORT.search(src))
|
||||||
|
wp = run_dir / "plugin" / "webpack.config.js"
|
||||||
|
wp_txt = wp.read_text(errors="replace") if wp.exists() else ""
|
||||||
|
externals_ok = "__TEABLE_HOST__" in wp_txt and "externals" in wp_txt
|
||||||
|
out.append(result("react 导入经 externals 映射(或无 react 裸导入)", externals_ok if uses_react else True,
|
||||||
|
"webpack externals: %s" % ("映射" if externals_ok else ("未映射!" if uses_react else "不需要"))))
|
||||||
|
out.append(result(f"definePlugin 注册且调用 ctx.{want_api}", "definePlugin" in src and want_api in src,
|
||||||
|
"definePlugin=%s %s=%s" % ("definePlugin" in src, want_api, want_api in src)))
|
||||||
|
if ev_name == "contextmenu-plugin":
|
||||||
|
edits = re.search(r"updateRecords|record.*update|OpenAPI", src, re.I)
|
||||||
|
out.append(result("实现了批量更新记录逻辑", bool(edits), edits.group(0) if edits else "未发现"))
|
||||||
|
else:
|
||||||
|
sel = re.search(r"useSelection|selection|getSelectionRecords", src, re.I)
|
||||||
|
out.append(result("通过宿主 SDK 获取选中记录", bool(sel), sel.group(0) if sel else "未发现"))
|
||||||
|
|
||||||
|
elif ev_name == "deploy-chain":
|
||||||
|
plugin_id = f"eval-deploy-{cfg_short}"
|
||||||
|
up = any("upload-bundle" in c for c in cmds)
|
||||||
|
out.append(result("调用了 upload-bundle 上传", up, [c for c in cmds if "upload-bundle" in c][:1]))
|
||||||
|
enabled = any("plugin-config" in c for c in cmds)
|
||||||
|
out.append(result("启用了 plugin-config", enabled, [c for c in cmds if "plugin-config" in c][:1]))
|
||||||
|
installed = any(re.search(r"context-menu.*install|plugin-context-menu", c) for c in cmds)
|
||||||
|
out.append(result("安装到表级 context-menu 位置", installed, [c for c in cmds if "context-menu" in c][:1]))
|
||||||
|
cleaned = any(("DELETE" in c.upper() and "/api/plugin/" in c) or re.search(r"plugin uninstall|plugin-install", c) for c in cmds) \
|
||||||
|
or (cli_json("plugin", "list") or [] and not any(p.get("id") == plugin_id for p in (cli_json("plugin", "list") or [])) and any("install" in c for c in cmds))
|
||||||
|
out.append(result("执行了清理(卸载安装/删除插件)", cleaned, [c for c in cmds if "uninstall" in c or "DELETE" in c.upper()][:2]))
|
||||||
|
lst = json.dumps(cli_json("plugin", "list") or [], ensure_ascii=False)
|
||||||
|
out.append(result("实例上无插件残留(plugin list 不含该 id)", plugin_id not in lst,
|
||||||
|
"残留!" if plugin_id in lst else "干净"))
|
||||||
|
verify = re.search(r"已确认|确认(安装|清理|删除)|installed|verified|清理(完成|干净)", ans)
|
||||||
|
out.append(result("答复确认了安装与清理结果", bool(verify), ans[:200]))
|
||||||
|
|
||||||
|
elif ev_name == "proxy-rule":
|
||||||
|
out.append(result("方案使用同源代理 /api/plugin-proxy/<service>/...", "/api/plugin-proxy/" in ans, ans[:250]))
|
||||||
|
out.append(result("指出需注册 PLUGIN_API_SERVICES 并重启 teable-backend", "PLUGIN_API_SERVICES" in ans,
|
||||||
|
"提及" if "PLUGIN_API_SERVICES" in ans else "未提及"))
|
||||||
|
lan_fetch = re.search(r"fetch\(\s*['\"]http://192\.168", ans)
|
||||||
|
out.append(result("不建议在 bundle 里直接请求局域网 IP", lan_fetch is None, lan_fetch.group(0) if lan_fetch else "ok"))
|
||||||
|
|
||||||
|
elif ev_name == "pitfall-diagnosis":
|
||||||
|
cause = re.search(r"(@teable/sdk|值导入|运行时导入|externals|undefined)|import", ans) and re.search(r"__TEABLE_HOST__|宿主全局|host global", ans)
|
||||||
|
out.append(result("定位为 @teable/* 运行时导入被 externals 映射为 undefined", bool(cause), ans[:250]))
|
||||||
|
out.append(result("给出修复:改从 window.__TEABLE_HOST__ 读取(或仅 import type)", "__TEABLE_HOST__" in ans, ans[:200]))
|
||||||
|
|
||||||
|
elif ev_name == "native-form-routing":
|
||||||
|
nf = re.search(r"NativeForm|useNativeForm|NativeFormScope", ans)
|
||||||
|
out.append(result("指出原生表单注入能力(NativeForm/NativeFormScope/useNativeForm)", bool(nf),
|
||||||
|
nf.group(0) if nf else ans[:200]))
|
||||||
|
# 咨询类任务应给出可实现方案(两档之一 + 关键代码/props)
|
||||||
|
plan = bool(nf) and bool(re.search(r"visibleFieldIds|slots|NativeFormField|formId|interceptSubmit", ans))
|
||||||
|
out.append(result("给出了具体实现要点(props/多步结构)", plan, ans[:250]))
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
iteration_dir = ROOT / sys.argv[1]
|
||||||
|
for eval_dir in sorted(iteration_dir.glob("eval-*")):
|
||||||
|
ev_name = eval_dir.name.split("-", 2)[2]
|
||||||
|
for config_dir in sorted(p for p in eval_dir.iterdir() if p.is_dir()):
|
||||||
|
run_dir = config_dir / "run-1"
|
||||||
|
if not (run_dir / "pi.jsonl").exists():
|
||||||
|
continue
|
||||||
|
cfg_short = "W" if config_dir.name == "with_skill" else "O"
|
||||||
|
try:
|
||||||
|
expectations = grade_eval(ev_name, run_dir, cfg_short)
|
||||||
|
except Exception as e:
|
||||||
|
expectations = [result("判分异常", False, repr(e))]
|
||||||
|
(run_dir / "grading.json").write_text(json.dumps({"expectations": expectations}, indent=2, ensure_ascii=False))
|
||||||
|
npass = sum(1 for x in expectations if x["passed"])
|
||||||
|
print(f"{eval_dir.name:34} {config_dir.name:12} {npass}/{len(expectations)}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"skill_name": "inkreach-teable-plugin",
|
||||||
|
"evals": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"name": "contextmenu-plugin",
|
||||||
|
"prompt": "帮我写一个右键菜单插件「批量置成功」:在表格里选中若干记录后右键,一键把这些记录的「结果」字段全部改成「成功」。把完整可构建的项目文件写到当前目录 plugin/ 子目录(plugin.json、webpack.config.js、src/…),手写即可,不必运行构建,也不用部署。",
|
||||||
|
"expected_output": "manifest 含 id/positions contextMenu;代码从 window.__TEABLE_HOST__ 取 SDK/OpenAPI/React;@teable/* 仅 import type 或不出现(禁止运行时值导入);definePlugin + ctx.addContextMenuAction;webpack externals 把 react 映射到 host 全局。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"name": "panel-widget",
|
||||||
|
"prompt": "写一个侧边面板插件「选中统计」:打开面板时实时显示当前表名和当前选中记录数量。项目文件写到 plugin/ 子目录,手写即可,不必构建部署。",
|
||||||
|
"expected_output": "positions panel;ctx.addPanel;选中数与表名来自 host SDK(useSelection/useTableId 等,经 window.__TEABLE_HOST__);无禁止的运行时导入。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"name": "deploy-chain",
|
||||||
|
"prompt": "当前目录 ready-plugin/ 下是我准备好的插件(plugin.json + dist/main.js)。把它部署到实例并安装到 {TBL} 这张表的右键菜单,确认安装成功;然后完整清理:卸载安装、停用、删除插件,确认都清干净。",
|
||||||
|
"expected_output": "upload-bundle(201)→ plugin-config 启用 → context-menu install → 验证;然后卸载安装 + DELETE /api/plugin/:id;plugin list 确认无残留。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"name": "proxy-rule",
|
||||||
|
"prompt": "我的插件需要调用我局域网的后端服务 http://192.168.124.92:8001/api/track 拿数据,实例是公网访问的(table.inkreach.cc),公网用户也要能用。插件里的请求应该怎么写?需要在实例侧改什么?给出说明和关键代码。",
|
||||||
|
"expected_output": "fetch('/api/plugin-proxy/<service>/api/track') 同源代理;注册 PLUGIN_API_SERVICES(两个 env 文件)+ 重启 teable-backend;不在 bundle 里硬编码局域网 IP;外部源需 manifest apiOrigins。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"name": "pitfall-diagnosis",
|
||||||
|
"prompt": "我上传的插件右键菜单能出现,但点击就白屏,控制台报:Cannot read properties of undefined (reading 'useFields')。帮我定位原因并给出修复。",
|
||||||
|
"expected_output": "定位为运行时从 @teable/sdk 等 import(被 webpack externals 映射成 undefined)/未从 window.__TEABLE_HOST__ 读取;修复=改用 host 全局取 SDK 或仅 import type。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"name": "native-form-routing",
|
||||||
|
"prompt": "我想在插件里做一个自定义布局的多步表单页,每一步嵌入系统原生的字段编辑器(不用自己重写输入组件),能做到吗?怎么实现?",
|
||||||
|
"expected_output": "指出用原生表单注入能力:NativeForm / NativeFormScope / useNativeForm(sdk-api.md 有专门章节);说明分层(Tier-1 注入/Tier-2 kit)。"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Launch inkreach-teable skill evals (iteration-4 A/B rerun).
|
||||||
|
|
||||||
|
Usage: python3 run_evals4.py <iteration-name> <tag> <config-spec> [eval-ids]
|
||||||
|
config-spec: with_skill=/root/eval-rerun/skill-new:/root/eval-rerun/bin-new/bin,old_skill=/root/eval-rerun/skill-old:/root/eval-rerun/bin-old/bin
|
||||||
|
|
||||||
|
Differences vs the original run_evals.py (eval-integrity fixes, see LESSONS.md):
|
||||||
|
- evals.json + sandbox-env.json live in THIS harness dir (outside the agent's cwd tree)
|
||||||
|
- per-config sandbox BASE via INKREACH_TEABLE_BASE_ID (no cross-config contamination)
|
||||||
|
- per-config PATH prepended with that config's CLI version dir (0.1.4 vs 0.1.5)
|
||||||
|
- pi runs with an isolated HOME (no user-level skills discovered)
|
||||||
|
- grading.json written only afterwards by grade_evals4.py (no answers near agents)
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
HARNESS = Path(__file__).parent # evals/ itself
|
||||||
|
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
|
||||||
|
MODEL = os.environ.get("EVAL_MODEL", "opencode-go/deepseek-v4.1-flash")
|
||||||
|
RUN_TIMEOUT = 600
|
||||||
|
PI_HOME = os.environ.get("EVAL_PI_HOME", "/root/eval-rerun/pi-home")
|
||||||
|
|
||||||
|
env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'sandbox-env.json'))).read_text())
|
||||||
|
|
||||||
|
PROMPT_TMPL = """你在操作一个已部署的 inkreach-teable 实例(Teable fork,自研自动化引擎)。
|
||||||
|
|
||||||
|
环境(已配置好,直接使用):
|
||||||
|
- CLI 命令: `inkreach-teable`(已在 PATH)。endpoint / token / base 已通过环境变量 INKREACH_TEABLE_ENDPOINT / INKREACH_TEABLE_TOKEN / INKREACH_TEABLE_BASE_ID 注入,不要执行 auth login,不要改配置文件。
|
||||||
|
- 只操作环境变量指定的这个基地(评估沙箱);严禁触碰其他基地。
|
||||||
|
- 工作目录就是当前目录,把中间产物写在这里。
|
||||||
|
|
||||||
|
{SKILL_LINE}
|
||||||
|
|
||||||
|
任务:{TASK}
|
||||||
|
|
||||||
|
要求:
|
||||||
|
1. 逐步执行任务,用 CLI 完成实际操作(不是只给命令)。
|
||||||
|
2. 完成后把最终答复写入 final_answer.md:做了什么、关键命令、结果、需要用户知道的事项。"""
|
||||||
|
|
||||||
|
|
||||||
|
def extract_final_text(jsonl_path: Path) -> str:
|
||||||
|
texts = []
|
||||||
|
for line in jsonl_path.read_text(errors="replace").splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
ev = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if ev.get("type") == "turn_end" and ev.get("message", {}).get("role") == "assistant":
|
||||||
|
for c in ev["message"].get("content", []):
|
||||||
|
if c.get("type") == "text" and c.get("text", "").strip():
|
||||||
|
texts.append(c["text"])
|
||||||
|
return texts[-1] if texts else ""
|
||||||
|
|
||||||
|
|
||||||
|
def extract_usage(jsonl_path: Path) -> dict:
|
||||||
|
usage = {}
|
||||||
|
for line in jsonl_path.read_text(errors="replace").splitlines():
|
||||||
|
m = re.search(r'"usage":\{[^{}]*"totalTokens":(\d+)', line)
|
||||||
|
if m:
|
||||||
|
usage["totalTokens"] = int(m.group(1))
|
||||||
|
return usage
|
||||||
|
|
||||||
|
|
||||||
|
def run_one(task, config, skill_path, bin_dir, cfg_env, run_dir: Path):
|
||||||
|
run_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
if config == "auto_skill":
|
||||||
|
# trigger test: no skill hint at all; the skill must be DISCOVERED from
|
||||||
|
# the user-level skills dir of the isolated (skilled) HOME.
|
||||||
|
skill_line = ""
|
||||||
|
elif skill_path:
|
||||||
|
skill_line = (
|
||||||
|
f"- 先完整阅读技能说明 {skill_path}/SKILL.md 并严格按其路由规则与关键约束操作;\n"
|
||||||
|
f" 需要细节时再按其中链接查阅 guides/ 与 api-reference/ 文档。"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
skill_line = "- 环境里没有额外文档,自行判断。"
|
||||||
|
prompt = PROMPT_TMPL.format(SKILL_LINE=skill_line, TASK=task)
|
||||||
|
(run_dir / "prompt.txt").write_text(prompt)
|
||||||
|
|
||||||
|
full_env = {
|
||||||
|
**os.environ,
|
||||||
|
"HOME": (os.environ.get("EVAL_PI_HOME_SKILLED", "/root/eval-rerun/pi-home-skilled") if config == "auto_skill" else PI_HOME),
|
||||||
|
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||||
|
**{k: env_all[k] for k in ("INKREACH_TEABLE_ENDPOINT", "INKREACH_TEABLE_TOKEN")},
|
||||||
|
**cfg_env,
|
||||||
|
}
|
||||||
|
cmd = ["timeout", str(RUN_TIMEOUT), "pi", "-p", "--mode", "json", "--model", MODEL, prompt]
|
||||||
|
t0 = time.time()
|
||||||
|
with open(run_dir / "pi.jsonl", "w") as out:
|
||||||
|
proc = subprocess.run(cmd, stdout=out, stderr=subprocess.DEVNULL, cwd=run_dir, env=full_env)
|
||||||
|
dur = time.time() - t0
|
||||||
|
timing = {"duration_ms": int(dur * 1000), "total_duration_seconds": round(dur, 1),
|
||||||
|
"total_tokens": extract_usage(run_dir / "pi.jsonl").get("totalTokens"),
|
||||||
|
"exit_code": proc.returncode, "cli": Path(bin_dir).name, "config": config}
|
||||||
|
(run_dir / "timing.json").write_text(json.dumps(timing, indent=2))
|
||||||
|
(run_dir / "assistant_final.md").write_text(extract_final_text(run_dir / "pi.jsonl") or "(no final text captured)")
|
||||||
|
return config, run_dir.name, proc.returncode, round(dur, 1)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
iteration = sys.argv[1]
|
||||||
|
tag = sys.argv[2]
|
||||||
|
iteration_dir = ROOT / iteration
|
||||||
|
configs = {}
|
||||||
|
for part in sys.argv[3].split(","):
|
||||||
|
name, _, rest = part.partition("=")
|
||||||
|
skill_path, _, bin_dir = rest.partition(":")
|
||||||
|
configs[name] = (skill_path, bin_dir)
|
||||||
|
|
||||||
|
evals = json.loads(Path(os.environ.get('EVAL_EVALS', str(HARNESS / 'teable-evals.json'))).read_text())["evals"]
|
||||||
|
if len(sys.argv) > 4 and sys.argv[4]:
|
||||||
|
only = {int(x) for x in sys.argv[4].split(",")}
|
||||||
|
evals = [e for e in evals if e["id"] in only]
|
||||||
|
|
||||||
|
jobs = []
|
||||||
|
for ev in evals:
|
||||||
|
name = f"eval-{ev['id']}-{ev['name']}"
|
||||||
|
for config, (skill_path, bin_dir) in configs.items():
|
||||||
|
cfg = env_all.get(config) or env_all['with_skill']
|
||||||
|
cfg_short = "W" if config == "with_skill" else "O"
|
||||||
|
task = (ev["prompt"]
|
||||||
|
.replace("{TAG}", f"{tag}{cfg_short}")
|
||||||
|
.replace("{WFID}", cfg["WF_ID"])
|
||||||
|
.replace("{TBL5}", cfg["TBL5_NAME"])
|
||||||
|
.replace("{ORD}", cfg["ORD"]))
|
||||||
|
meta = {"eval_id": ev["id"], "eval_name": ev["name"], "config": config, "prompt": task, "assertions": []}
|
||||||
|
rd = iteration_dir / name / config / "run-1"
|
||||||
|
rd.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
(rd.parent / "eval_metadata.json").write_text(json.dumps(meta, indent=2, ensure_ascii=False))
|
||||||
|
cfg_env = {
|
||||||
|
"INKREACH_TEABLE_BASE_ID": cfg["INKREACH_TEABLE_BASE_ID"],
|
||||||
|
"INKREACH_TEABLE_EVAL_TBL": cfg["TBL5_ID"],
|
||||||
|
"INKREACH_TEABLE_EVAL_ORD": cfg["ORD"],
|
||||||
|
}
|
||||||
|
if ev["id"] == 12:
|
||||||
|
cfg_env["INKREACH_TEABLE_TOKEN"] = env_all["LOWPERM_TOKEN"]
|
||||||
|
jobs.append((task, config, skill_path, bin_dir, cfg_env, rd))
|
||||||
|
|
||||||
|
print(f"{len(jobs)} runs -> {iteration_dir}", flush=True)
|
||||||
|
with ThreadPoolExecutor(max_workers=6) as ex:
|
||||||
|
futs = [ex.submit(run_one, *j) for j in jobs]
|
||||||
|
for f in futs:
|
||||||
|
print(f.result(), flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Multi-turn evals: two-round conversations where turn 2 corrects turn 1.
|
||||||
|
|
||||||
|
Verifies the agent AMENDS the previous artifacts (edit workflow / add field)
|
||||||
|
instead of ignoring the correction or rebuilding blindly.
|
||||||
|
|
||||||
|
Usage: python3 run_multiturn.py <iteration> <tag>
|
||||||
|
"""
|
||||||
|
import json, os, subprocess, sys, time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
HARNESS = Path(__file__).parent
|
||||||
|
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
|
||||||
|
MODEL = os.environ.get("EVAL_MODEL", "opencode-go/deepseek-v4.1-flash")
|
||||||
|
RUN_TIMEOUT = 480
|
||||||
|
PI_HOME = os.environ.get("EVAL_PI_HOME_SKILLED", "/root/eval-rerun/pi-home-skilled") # trigger mode: skill auto-discovered
|
||||||
|
|
||||||
|
env_all = json.loads((HARNESS / "sandbox-env.json").read_text())
|
||||||
|
|
||||||
|
TASKS = [
|
||||||
|
{
|
||||||
|
"name": "mt-1-notify-correction",
|
||||||
|
"turn1": "帮我建个自动化:order_logs 表有新记录的时候通知我。名字你来定,配好激活。",
|
||||||
|
"turn2": "不对,改一下:别用邮件,要站内通知;而且只在「结果」字段变成「失败」的时候才通知,不是有新记录就通知。",
|
||||||
|
"turn1_hint": None,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "mt-2-incremental-schema",
|
||||||
|
"turn1": "帮我建张表存客户反馈,加个单行文本「内容」字段,随便插一条示例。",
|
||||||
|
"turn2": "再补一个单选字段「渠道」,选项:微信/电话/邮件。然后把刚才那条示例记录的渠道标成「微信」。",
|
||||||
|
"turn1_hint": None,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
WRAP = """要求:
|
||||||
|
1. 逐步用 CLI 完成实际操作(不是只给命令)。
|
||||||
|
2. 每轮结束后把当前轮的答复写入 final_answer_turn{N}.md(做了什么、关键命令、结果)。
|
||||||
|
3. 用户的后续修正优先于之前的做法;能改就改现有的,不要无视."""
|
||||||
|
|
||||||
|
|
||||||
|
def turn(run_dir: Path, sid: str, text: str, n: int):
|
||||||
|
prompt = f"{text}\n\n{WRAP.format(N=n)}"
|
||||||
|
cmd = ["timeout", str(RUN_TIMEOUT), "pi", "-p", "--mode", "json",
|
||||||
|
"--session-id", sid, "--model", MODEL, prompt]
|
||||||
|
out_f = run_dir / f"turn{n}.jsonl"
|
||||||
|
env = {
|
||||||
|
**os.environ, "HOME": PI_HOME,
|
||||||
|
"PATH": f"/root/eval-rerun/bin-new/bin:{os.environ['PATH']}",
|
||||||
|
"INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"],
|
||||||
|
"INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"],
|
||||||
|
"INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"],
|
||||||
|
}
|
||||||
|
with open(out_f, "w") as out:
|
||||||
|
p = subprocess.run(cmd, stdout=out, stderr=subprocess.DEVNULL, cwd=run_dir, env=env)
|
||||||
|
# pull last assistant text
|
||||||
|
last = ""
|
||||||
|
for line in out_f.read_text(errors="replace").splitlines():
|
||||||
|
try:
|
||||||
|
ev = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if ev.get("type") == "turn_end" and ev.get("message", {}).get("role") == "assistant":
|
||||||
|
for c in ev["message"].get("content", []):
|
||||||
|
if c.get("type") == "text" and c.get("text", "").strip():
|
||||||
|
last = c["text"]
|
||||||
|
(run_dir / f"final_answer_turn{n}.md").write_text(last or "(none)")
|
||||||
|
return p.returncode
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
iteration, tag = sys.argv[1], sys.argv[2]
|
||||||
|
for t in TASKS:
|
||||||
|
rd = ROOT / iteration / t["name"] / "auto_skill" / "run-1"
|
||||||
|
rd.mkdir(parents=True, exist_ok=True)
|
||||||
|
sid = f"mt-{tag}-{t['name']}"
|
||||||
|
t0 = time.time()
|
||||||
|
r1 = turn(rd, sid, t["turn1"], 1)
|
||||||
|
r2 = turn(rd, sid, t["turn2"], 2)
|
||||||
|
print(f"{t['name']}: turn1 exit={r1} turn2 exit={r2} dur={round(time.time()-t0,1)}s", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Launch inkreach-teable-plugin skill evals (integrity-clean port).
|
||||||
|
|
||||||
|
Usage: python3 run_plugin_evals.py <iteration> <tag> <config-spec> [eval-ids]
|
||||||
|
config-spec: with_skill=/root/eval-rerun/skill-plugin-new:/root/eval-rerun/bin-new/bin
|
||||||
|
"""
|
||||||
|
import json, os, re, shutil, subprocess, sys, time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
HARNESS = Path(__file__).parent # evals/ itself
|
||||||
|
ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun"))
|
||||||
|
MODEL = os.environ.get("EVAL_MODEL", "opencode-go/deepseek-v4.1-flash")
|
||||||
|
RUN_TIMEOUT = 900 # plugin evals build bundles — need more headroom
|
||||||
|
PI_HOME = os.environ.get("EVAL_PI_HOME", "/root/eval-rerun/pi-home")
|
||||||
|
FIXTURE = "/root/pi-skill-lab/plugin-workspace/fixture-deploy-W"
|
||||||
|
|
||||||
|
env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'plugin-sandbox-env.json'))).read_text())
|
||||||
|
|
||||||
|
PROMPT_TMPL = """你在为一套已部署的 inkreach-teable 实例(Teable fork)开发/部署动态 React 插件。
|
||||||
|
|
||||||
|
环境(已配置好,直接使用):
|
||||||
|
- 后端 http://localhost:3003,前端 http://localhost:3002。PAT 已通过环境变量 INKREACH_TEABLE_ENDPOINT / INKREACH_TEABLE_TOKEN 注入(`inkreach-teable` CLI 可直接用)。
|
||||||
|
- 评估沙箱基地 ID:{BASE}。凡涉及实例的读写一律只在这个基地内,严禁触碰其他基地;严禁向实例的插件注册表上传与任务无关的演示插件。
|
||||||
|
- 工作目录就是当前目录,所有产出写在这里。
|
||||||
|
|
||||||
|
{SKILL_LINE}
|
||||||
|
|
||||||
|
任务:{TASK}
|
||||||
|
|
||||||
|
要求:
|
||||||
|
1. 按任务性质完成:咨询类问题(能不能/怎么做)直接给出方案与关键代码即可,不必搭建完整工程;开发/部署类任务才需要实际构建与部署。
|
||||||
|
2. 完成后把最终答复写入 final_answer.md:结论、关键实现/命令、结果、需要用户知道的事项。"""
|
||||||
|
|
||||||
|
|
||||||
|
def extract_final_text(p: Path) -> str:
|
||||||
|
texts = []
|
||||||
|
for line in p.read_text(errors="replace").splitlines():
|
||||||
|
try:
|
||||||
|
ev = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if ev.get("type") == "turn_end" and ev.get("message", {}).get("role") == "assistant":
|
||||||
|
for c in ev["message"].get("content", []):
|
||||||
|
if c.get("type") == "text" and c.get("text", "").strip():
|
||||||
|
texts.append(c["text"])
|
||||||
|
return texts[-1] if texts else ""
|
||||||
|
|
||||||
|
|
||||||
|
def run_one(task, config, skill_path, bin_dir, cfg_env, run_dir: Path):
|
||||||
|
run_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
skill_line = (
|
||||||
|
f"- 先完整阅读技能说明 {skill_path}/SKILL.md 并按其模块地图(按需读 dynamic-plugins/ 下文档)操作。"
|
||||||
|
if skill_path else ""
|
||||||
|
)
|
||||||
|
prompt = PROMPT_TMPL.format(BASE=cfg_env["INKREACH_TEABLE_BASE_ID"], SKILL_LINE=skill_line, TASK=task)
|
||||||
|
(run_dir / "prompt.txt").write_text(prompt)
|
||||||
|
full_env = {
|
||||||
|
**os.environ, "HOME": PI_HOME, "PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||||
|
**{k: env_all[k] for k in ("INKREACH_TEABLE_ENDPOINT", "INKREACH_TEABLE_TOKEN")},
|
||||||
|
**cfg_env,
|
||||||
|
}
|
||||||
|
cmd = ["timeout", str(RUN_TIMEOUT), "pi", "-p", "--mode", "json", "--model", MODEL, prompt]
|
||||||
|
t0 = time.time()
|
||||||
|
with open(run_dir / "pi.jsonl", "w") as out:
|
||||||
|
proc = subprocess.run(cmd, stdout=out, stderr=subprocess.DEVNULL, cwd=run_dir, env=full_env)
|
||||||
|
(run_dir / "timing.json").write_text(json.dumps(
|
||||||
|
{"duration_s": round(time.time() - t0, 1), "exit_code": proc.returncode, "config": config}))
|
||||||
|
(run_dir / "assistant_final.md").write_text(extract_final_text(run_dir / "pi.jsonl") or "(none)")
|
||||||
|
return config, run_dir.name, proc.returncode
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
iteration, tag = sys.argv[1], sys.argv[2]
|
||||||
|
iteration_dir = ROOT / iteration
|
||||||
|
configs = {}
|
||||||
|
for part in sys.argv[3].split(","):
|
||||||
|
name, _, rest = part.partition("=")
|
||||||
|
sp, _, bd = rest.partition(":")
|
||||||
|
configs[name] = (sp, bd)
|
||||||
|
evals = json.loads(Path(os.environ.get('EVAL_EVALS', str(HARNESS / 'plugin-evals.json'))).read_text())["evals"]
|
||||||
|
if len(sys.argv) > 4 and sys.argv[4]:
|
||||||
|
only = {int(x) for x in sys.argv[4].split(",")}
|
||||||
|
evals = [e for e in evals if e["id"] in only]
|
||||||
|
|
||||||
|
jobs = []
|
||||||
|
for ev in evals:
|
||||||
|
name = f"eval-{ev['id']}-{ev['name']}"
|
||||||
|
for config, (skill_path, bin_dir) in configs.items():
|
||||||
|
cfg = env_all[config]
|
||||||
|
cfg_short = "W" if config == "with_skill" else "O"
|
||||||
|
task = ev["prompt"].replace("{TBL}", cfg["TBL"])
|
||||||
|
rd = iteration_dir / name / config / "run-1"
|
||||||
|
rd.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
# eval-3 needs the ready-plugin fixture bundle in cwd
|
||||||
|
if ev["id"] == 3:
|
||||||
|
rp = rd / "ready-plugin"
|
||||||
|
if rp.exists():
|
||||||
|
shutil.rmtree(rp)
|
||||||
|
shutil.copytree(FIXTURE, rp)
|
||||||
|
(rd.parent / "eval_metadata.json").write_text(json.dumps(
|
||||||
|
{"eval_id": ev["id"], "prompt": task, "assertions": []}, ensure_ascii=False))
|
||||||
|
cfg_env = {"INKREACH_TEABLE_BASE_ID": cfg["INKREACH_TEABLE_BASE_ID"]}
|
||||||
|
jobs.append((task, config, skill_path, bin_dir, cfg_env, rd))
|
||||||
|
|
||||||
|
print(f"{len(jobs)} runs -> {iteration_dir}", flush=True)
|
||||||
|
with ThreadPoolExecutor(max_workers=3) as ex: # builds are heavy — 3 workers
|
||||||
|
for f in [ex.submit(run_one, *j) for j in jobs]:
|
||||||
|
print(f.result(), flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
{
|
||||||
|
"skill_name": "inkreach-teable",
|
||||||
|
"evals": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"name": "button-click-automation",
|
||||||
|
"prompt": "帮我在 order_logs 表里加一个按钮字段(名字用:催单按钮{TAG}),点这个按钮的时候触发一个自动化,自动化里跑一段脚本,脚本返回当前记录的订单号。然后激活它,创建一条新记录并点一下按钮测试,确认执行结果。",
|
||||||
|
"expected_output": "创建 button 字段(options.workflow 为对象 {id,...})、workflow、buttonClick 触发器、读取 triggerInput 的 script action、activate、fire button-click 测试、runs 确认 succeeded。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"name": "scheduled-digest",
|
||||||
|
"prompt": "我想要每天定时跑一次自动化(工作流名字用:每日统计{TAG}),统计 order_logs 表里的记录数并输出。帮我配好。",
|
||||||
|
"expected_output": "scheduledTime 触发器只配 timing.type + timing.interval(后端忽略其他时间字段);script/getRecords action;activate。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"name": "record-condition-email",
|
||||||
|
"prompt": "当 order_logs 表里有记录的「结果」字段变成「失败」时,自动发邮件通知 ops@xingchen.tech(工作流名字用:失败告警{TAG})。帮我配好。",
|
||||||
|
"expected_output": "recordCondition 触发器(分类是 recordCondition,条件键名是 conditions 而非 filter);sendEmail action 收件人 ops@xingchen.tech;activate。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"name": "debug-failed-run",
|
||||||
|
"prompt": "我有个自动化 wfl5F8dQRDloo28w909……不对,ID 是 {WFID},名字叫 nightly-sync,最近好像没正常跑,帮我看看是什么情况,别急着改,先诊断清楚再说。",
|
||||||
|
"expected_output": "先 workflow get 查 isActive → runs list --status failed → runs get 看 nodeResults 错误;指出脚本抛错(boom: upstream sync timeout);未盲目重新激活。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"name": "select-constraint-write",
|
||||||
|
"prompt": "在 {TBL5} 表里,把订单号为 {ORD} 的那条记录的「结果」改成「已完成」。另外告诉我这个「结果」字段目前一共有哪些可选值。",
|
||||||
|
"expected_output": "先查约束发现「已完成」不在 choices;要么说明并停下,要么先加选项再写入;不发生静默失败。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"name": "official-cli-near-miss",
|
||||||
|
"prompt": "用 teable 的 CLI 工具帮我列出这个基地里都有哪些表。",
|
||||||
|
"expected_output": "路由到 inkreach-teable CLI(官方 teable CLI 在该实例自动化契约不兼容);列出 order_logs;未安装/调用官方 teable CLI 打实例。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"name": "webhook-receiver",
|
||||||
|
"prompt": "帮我建一个自动化(名字:回单通知{TAG}),用 webhook 触发:外部系统会 POST 调用它。建好激活后,把可以调用的完整 URL 给我,并实际测试一次确认能通。",
|
||||||
|
"expected_output": "webhook 触发器;URL 形如 {endpoint}/api/webhook/{baseId}/{workflowId};用 fire webhook 或 curl 实测;已激活。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 8,
|
||||||
|
"name": "weekday-schedule-trap",
|
||||||
|
"prompt": "我要一个自动化(名字:工作日盘点{TAG}),每个工作日下午5点跑一次,统计 order_logs 表的记录条数就行。帮我配好。",
|
||||||
|
"expected_output": "知道后端只认 timing.type+interval:要么明确告知『具体时刻/星期』不被支持并给出周期间隔替代方案;绝不悄悄配 weekdays/triggerTime 假装生效。timing 配置无被忽略字段。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 9,
|
||||||
|
"name": "secret-env-usage",
|
||||||
|
"prompt": "建一个自动化(名字:汇率查询{TAG}),脚本里调用外部汇率 API,API key 是 sk-test-abc123,不要明文写在脚本里。配好激活即可,不用真跑通外部接口。",
|
||||||
|
"expected_output": "secret set 存 key(如 EXCHANGE_KEY)→ 脚本读 process.env.EXCHANGE_KEY;代码中不出现明文 key;激活。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 10,
|
||||||
|
"name": "script-npm-deps",
|
||||||
|
"prompt": "我有个自动化脚本要用 dayjs 处理日期格式化。帮我建个演示工作流(名字:日期演示{TAG}),脚本里 require dayjs 并格式化当前日期输出,把依赖正确声明好。",
|
||||||
|
"expected_output": "action add/update 带 --dependencies '[{\"name\":\"dayjs\",\"version\":...}]'(或 dayjs@版本);知晓后端需 npm 在 service PATH(风险提示)。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 11,
|
||||||
|
"name": "field-options-update",
|
||||||
|
"prompt": "把 order_logs 表「结果」字段加一个新选项「已取消」。就这一件事。",
|
||||||
|
"expected_output": "field update --options(走 convert API;普通 PATCH 会丢 options);验证 choices 里出现「已取消」。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 12,
|
||||||
|
"name": "lowperm-403-handling",
|
||||||
|
"prompt": "帮我在当前基地里建一张新表,名字叫快递跟踪,加个单行文本字段 运单号,再加个数字字段 金额。建好后随便插一条示例数据。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 13,
|
||||||
|
"name": "deep-pagination",
|
||||||
|
"prompt": "big_logs 这张表里记录挺多的。帮我把订单号以 SLOW- 开头的记录全部找出来:一共几条?再挑其中一条,把它的完整字段值原样发我。"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 14,
|
||||||
|
"name": "vague-daily-notify",
|
||||||
|
"prompt": "我想大概每天知道一下订单进展。邮件先不用,站内通知就行。你看着帮我弄一个。"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@inkreach/inkreach-teable",
|
"name": "@inkreach/inkreach-teable",
|
||||||
"version": "0.1.8",
|
"version": "0.1.9",
|
||||||
"description": "CLI (command: inkreach-teable) for operating inkreach-teable: tables, fields, records, views, and the self-built automation engine (8 triggers + 7 actions).",
|
"description": "CLI (command: inkreach-teable) for operating inkreach-teable: tables, fields, records, views, and the self-built automation engine (8 triggers + 7 actions).",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"type": "commonjs",
|
"type": "commonjs",
|
||||||
|
|||||||
@@ -51,8 +51,12 @@ cleanup() { curl -s -o /dev/null -X DELETE -H "Authorization: Bearer $TOK" "$EP/
|
|||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
echo "sandbox base: $BASE"
|
echo "sandbox base: $BASE"
|
||||||
|
|
||||||
# ---------- auth / base ----------
|
# ---------- auth / base / space ----------
|
||||||
smoke "auth status" $CLI auth status
|
smoke "auth status" $CLI auth status
|
||||||
|
smoke "space list" $CLI space list
|
||||||
|
SB2=$($CLI base create --space-id "$SPACE" --name cli-smoke-b2 --json | j "d['id']")
|
||||||
|
smoke "base create (CLI)" $CLI base get "$SB2"
|
||||||
|
smoke "base delete (CLI)" $CLI base delete "$SB2"
|
||||||
smoke "auth token list" $CLI auth token list
|
smoke "auth token list" $CLI auth token list
|
||||||
smoke "base list" $CLI base list
|
smoke "base list" $CLI base list
|
||||||
smoke "base get" $CLI base get "$BASE"
|
smoke "base get" $CLI base get "$BASE"
|
||||||
@@ -176,13 +180,46 @@ smoke "plugin list" $CLI plugin list
|
|||||||
smoke "plugin installs" $CLI plugin installs
|
smoke "plugin installs" $CLI plugin installs
|
||||||
smoke "plugin csp list" $CLI plugin csp list
|
smoke "plugin csp list" $CLI plugin csp list
|
||||||
|
|
||||||
# ---------- auth token create/delete (self-cleaning PAT) ----------
|
# ---------- auth token routes (via low-priv account session; PAT cannot mint PATs) ----------
|
||||||
PATID=$($CLI auth token create --name cli-smoke --json 2>/dev/null | j "d['id']") || PATID=""
|
# Provide SMOKE_LOWPRIV_EMAIL / SMOKE_LOWPRIV_PASSWORD (any account you may sign in).
|
||||||
if [ -n "${PATID:-}" ]; then
|
if [ -n "${SMOKE_LOWPRIV_EMAIL:-}" ] && [ -n "${SMOKE_LOWPRIV_PASSWORD:-}" ]; then
|
||||||
smoke "auth token create" $CLI auth token list
|
curl -s -c /tmp/cli-smoke-cookie.txt "$EP/api/auth/signin" -X POST -H 'Content-Type: application/json' \
|
||||||
smoke "auth token delete" $CLI auth token delete "$PATID"
|
-d "{\"email\":\"$SMOKE_LOWPRIV_EMAIL\",\"password\":\"$SMOKE_LOWPRIV_PASSWORD\"}" >/dev/null
|
||||||
|
PATID=$(curl -s -b /tmp/cli-smoke-cookie.txt -X POST "$EP/api/access-token" -H 'Content-Type: application/json' \
|
||||||
|
-d '{"name":"cli-smoke","scopes":["table|read"],"expiredTime":"2027-01-01T00:00:00.000Z"}' \
|
||||||
|
| python3 -c "import sys,json;print(json.load(sys.stdin).get('id',''))")
|
||||||
|
if [ -n "$PATID" ]; then
|
||||||
|
CODE=$(curl -s -o /dev/null -w '%{http_code}' -b /tmp/cli-smoke-cookie.txt -X DELETE "$EP/api/access-token/$PATID")
|
||||||
|
if [ "$CODE" = "200" ]; then PASS=$((PASS+1)); echo "pass auth token create+delete (REST, session)"; else FAIL=$((FAIL+1)); echo "FAIL auth token delete (HTTP $CODE)"; fi
|
||||||
else
|
else
|
||||||
echo "warn auth token create returned no id (may need scopes) — check manually"
|
echo "warn auth token create via session failed — check creds"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "warn auth-token routes not covered (set SMOKE_LOWPRIV_EMAIL/PASSWORD to enable)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ---------- plugin chain (opt-in: SMOKE_PLUGINS=1; uploads to the registry, cleaned after) ----------
|
||||||
|
if [ "${SMOKE_PLUGINS:-0}" = "1" ] && [ -f /root/pi-skill-lab/plugin-workspace/fixture-deploy-W/plugin.json ]; then
|
||||||
|
FIXT=/root/pi-skill-lab/plugin-workspace/fixture-deploy-W
|
||||||
|
PID2=$(python3 -c "import json;print(json.load(open('$FIXT/plugin.json'))['id'])")
|
||||||
|
UP=$(curl -s -b "auth_session=unused" -H "Authorization: Bearer $TOK" \
|
||||||
|
-F "js=@$FIXT/dist/main.js" -F "manifest=@$FIXT/plugin.json" \
|
||||||
|
"$EP/api/plugin/$PID2/upload-bundle" | python3 -c "import sys,json;print(json.load(sys.stdin).get('id',''))" 2>/dev/null)
|
||||||
|
if [ -n "$UP" ]; then
|
||||||
|
smoke "plugin upload-bundle" $CLI plugin list
|
||||||
|
smoke "plugin-config enable" curl -sf -X PUT -H "Authorization: Bearer $TOK" -H 'Content-Type: application/json' \
|
||||||
|
-d '{"enabled":true,"tableIds":[]}' "$EP/api/base/$BASE/plugin-config/$PID2" -o /dev/null
|
||||||
|
smoke "plugin install" $CLI plugin install --plugin "$PID2" --name cli-smoke --position base
|
||||||
|
IID=$($CLI plugin installs --json 2>/dev/null | python3 -c "
|
||||||
|
import sys,json
|
||||||
|
for i in (json.load(sys.stdin) or []):
|
||||||
|
if i.get('pluginId')=='$PID2': print(i.get('id') or i.get('installId','')); break" 2>/dev/null)
|
||||||
|
[ -n "$IID" ] && smoke "plugin uninstall" $CLI plugin uninstall --install "$IID"
|
||||||
|
CODE=$(curl -s -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $TOK" "$EP/api/plugin/$PID2")
|
||||||
|
if [ "$CODE" = "200" ]; then PASS=$((PASS+1)); echo "pass plugin registry cleanup"; else FAIL=$((FAIL+1)); echo "FAIL plugin delete (HTTP $CODE)"; fi
|
||||||
|
else
|
||||||
|
echo "warn plugin upload failed — PAT lacks register rights: POST /api/plugin and DELETE /api/plugin/:id are admin-only (403); upload-bundle alone needs the plugin_developer whitelist. Provide an admin session to cover the full chain."
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ---------- table delete (destructive, scratch only) ----------
|
# ---------- table delete (destructive, scratch only) ----------
|
||||||
|
|||||||
@@ -29,6 +29,12 @@ Tokens (`Authorization: Bearer <PAT>`).
|
|||||||
**Returns** `201` with `{ id, name, version, ... }` — the plugin ID to use for
|
**Returns** `201` with `{ id, name, version, ... }` — the plugin ID to use for
|
||||||
enable and install.
|
enable and install.
|
||||||
|
|
||||||
|
**Registering a NEW plugin id**: `POST /api/plugin` (manifest as JSON body) —
|
||||||
|
**admin-only** (PAT gets 403 `restricted_resource`). Uploading a bundle for an
|
||||||
|
already-registered id only needs the `plugin_developer` whitelist. Deleting a
|
||||||
|
plugin (`DELETE /api/plugin/:id`) is likewise admin-only. PAT-only automation
|
||||||
|
should reuse existing registered ids, or ask an admin to register/delete.
|
||||||
|
|
||||||
**Security** (7-layer validation):
|
**Security** (7-layer validation):
|
||||||
1. Authentication — valid session cookie required (401 if missing)
|
1. Authentication — valid session cookie required (401 if missing)
|
||||||
2. Developer whitelist — `plugin_developer` table lookup (403 if not whitelisted)
|
2. Developer whitelist — `plugin_developer` table lookup (403 if not whitelisted)
|
||||||
|
|||||||
@@ -9,6 +9,13 @@ description: >-
|
|||||||
Operate a deployed inkreach-teable instance via the `inkreach-teable` CLI.
|
Operate a deployed inkreach-teable instance via the `inkreach-teable` CLI.
|
||||||
Respond in the user's language. Keep answers concise and action-oriented.
|
Respond in the user's language. Keep answers concise and action-oriented.
|
||||||
|
|
||||||
|
## 0. 执行守则
|
||||||
|
|
||||||
|
- 多步任务先规划再动手;**完成全部步骤后才给最终答复**,不要中途停止或只给命令不给结果。
|
||||||
|
- 完成后如实汇报:做了什么、关键命令、结果;**做不到就明说做不到及原因,绝不谎报成功**。
|
||||||
|
- 长任务(批量记录、多工作流)分批执行,每批确认后再继续。
|
||||||
|
- **改错了怎么撤销**:workflow 的 snapshot 仅供审计,**没有恢复 API**——用 `workflow deactivate` 止损,再用 `trigger set` / `action update` 改回(节点更新是全图替换,直接改回正确配置即可);整条不要了用 `workflow delete`(软删)。
|
||||||
|
|
||||||
> **Critical**: the automation engine is **self-built and differs from official
|
> **Critical**: the automation engine is **self-built and differs from official
|
||||||
> teable** — the official `teable` CLI corrupts or 404s on automations here.
|
> teable** — the official `teable` CLI corrupts or 404s on automations here.
|
||||||
> Exact key-name differences are in [Key Constraints](#3-key-constraints).
|
> Exact key-name differences are in [Key Constraints](#3-key-constraints).
|
||||||
|
|||||||
@@ -27,6 +27,15 @@ inkreach-teable auth token delete <id>
|
|||||||
|
|
||||||
- `auth token create` returns the full token **only once**. Default scopes: `automation|read,automation|create,automation|update,automation|delete,table|read,record|read,record|create,record|update`.
|
- `auth token create` returns the full token **only once**. Default scopes: `automation|read,automation|create,automation|update,automation|delete,table|read,record|read,record|create,record|update`.
|
||||||
|
|
||||||
|
## space / base
|
||||||
|
|
||||||
|
```bash
|
||||||
|
inkreach-teable space list
|
||||||
|
inkreach-teable space create --name "<name>"
|
||||||
|
inkreach-teable base create --space-id <spaceId> --name "<name>"
|
||||||
|
inkreach-teable base delete <baseId> # DESTRUCTIVE: soft-deletes all tables in it
|
||||||
|
```
|
||||||
|
|
||||||
## base / table / field / record / view
|
## base / table / field / record / view
|
||||||
|
|
||||||
These resources use the official-teable-compatible API. Full guide: [tables-fields.md](tables-fields.md).
|
These resources use the official-teable-compatible API. Full guide: [tables-fields.md](tables-fields.md).
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ async function fieldNameMap(client: Client, tableId: string): Promise<Record<str
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function baseResourceCommands(): Command[] {
|
export function baseResourceCommands(): Command[] {
|
||||||
return [baseCmd(), tableCmd(), fieldCmd(), recordCmd(), viewCmd(), presetCmd()];
|
return [baseCmd(), spaceCmd(), tableCmd(), fieldCmd(), recordCmd(), viewCmd(), presetCmd()];
|
||||||
}
|
}
|
||||||
|
|
||||||
// ===================== base =====================
|
// ===================== base =====================
|
||||||
@@ -82,6 +82,35 @@ function baseCmd(): Command {
|
|||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
|
cmd
|
||||||
|
.command('create')
|
||||||
|
.description('Create a base in a space')
|
||||||
|
.requiredOption('--space-id <id>', 'Space id (see `space list`)')
|
||||||
|
.requiredOption('--name <name>', 'New base name')
|
||||||
|
.action(
|
||||||
|
run(async (ctx, _a, o) => {
|
||||||
|
const opts = o as { spaceId: string; name: string };
|
||||||
|
const base = await ctx.client.createBase(opts.spaceId, opts.name);
|
||||||
|
if (ctx.json) printJson(base);
|
||||||
|
else {
|
||||||
|
const b = base as Record<string, unknown>;
|
||||||
|
printKv([['id', b.id], ['name', b.name]]);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
cmd
|
||||||
|
.command('delete <baseId>')
|
||||||
|
.description('Soft-delete a base (DESTRUCTIVE: hides all its tables)')
|
||||||
|
.action(
|
||||||
|
run(async (ctx, args) => {
|
||||||
|
const baseId = args[0] as string;
|
||||||
|
await ctx.client.deleteBase(baseId);
|
||||||
|
if (ctx.json) printJson({ ok: true, id: baseId });
|
||||||
|
else process.stdout.write(`✓ Base ${baseId} deleted (soft)\n`);
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
cmd
|
cmd
|
||||||
.command('get <baseId>')
|
.command('get <baseId>')
|
||||||
.description('Get base details')
|
.description('Get base details')
|
||||||
@@ -100,6 +129,54 @@ function baseCmd(): Command {
|
|||||||
return cmd;
|
return cmd;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ===================== space =====================
|
||||||
|
|
||||||
|
function spaceCmd(): Command {
|
||||||
|
const cmd = new Command('space');
|
||||||
|
cmd.description('Manage spaces (top-level containers for bases)');
|
||||||
|
|
||||||
|
cmd
|
||||||
|
.command('list')
|
||||||
|
.description('List spaces you can access')
|
||||||
|
.action(
|
||||||
|
run(async (ctx) => {
|
||||||
|
const list = (await ctx.client.listSpaces()) as Array<Record<string, unknown>>;
|
||||||
|
if (ctx.json) printJson(list);
|
||||||
|
else {
|
||||||
|
if (!list.length) {
|
||||||
|
process.stdout.write('No spaces found.\n');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
printTable(
|
||||||
|
[
|
||||||
|
{ key: 'id', header: 'ID' },
|
||||||
|
{ key: 'name', header: 'NAME' },
|
||||||
|
],
|
||||||
|
list.map((sp) => ({ id: truncate(sp.id), name: truncate(sp.name, 40) }))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
cmd
|
||||||
|
.command('create')
|
||||||
|
.description('Create a space')
|
||||||
|
.requiredOption('--name <name>', 'New space name')
|
||||||
|
.action(
|
||||||
|
run(async (ctx, _a, o) => {
|
||||||
|
const opts = o as { name: string };
|
||||||
|
const sp = await ctx.client.createSpace(opts.name);
|
||||||
|
if (ctx.json) printJson(sp);
|
||||||
|
else {
|
||||||
|
const s2 = sp as Record<string, unknown>;
|
||||||
|
printKv([['id', s2.id], ['name', s2.name]]);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
return cmd;
|
||||||
|
}
|
||||||
|
|
||||||
// ===================== table =====================
|
// ===================== table =====================
|
||||||
|
|
||||||
function tableCmd(): Command {
|
function tableCmd(): Command {
|
||||||
|
|||||||
@@ -54,6 +54,18 @@ export class Client {
|
|||||||
getBase(baseId: string) {
|
getBase(baseId: string) {
|
||||||
return this.call<unknown>('GET', `/api/base/${baseId}`);
|
return this.call<unknown>('GET', `/api/base/${baseId}`);
|
||||||
}
|
}
|
||||||
|
createBase(spaceId: string, name: string) {
|
||||||
|
return this.call<unknown>('POST', '/api/base', { spaceId, name });
|
||||||
|
}
|
||||||
|
deleteBase(baseId: string) {
|
||||||
|
return this.call<{ id: string }>('DELETE', `/api/base/${baseId}`);
|
||||||
|
}
|
||||||
|
listSpaces() {
|
||||||
|
return this.call<unknown[]>('GET', '/api/space');
|
||||||
|
}
|
||||||
|
createSpace(name: string) {
|
||||||
|
return this.call<unknown>('POST', '/api/space', { name });
|
||||||
|
}
|
||||||
|
|
||||||
// ---- table ----
|
// ---- table ----
|
||||||
listTables(baseId: string) {
|
listTables(baseId: string) {
|
||||||
|
|||||||
@@ -28,6 +28,10 @@ export function buildHint(status: number, message: string): string | undefined {
|
|||||||
return '违反了字段的 visibilityFilter（行级条件）—— `inkreach-teable field visible-choices <tableId> <recordId> <field>` 查看该行可见选项。';
|
return '违反了字段的 visibilityFilter（行级条件）—— `inkreach-teable field visible-choices <tableId> <recordId> <field>` 查看该行可见选项。';
|
||||||
if (/expected array, received/i.test(m))
|
if (/expected array, received/i.test(m))
|
||||||
return '参数形状不符 —— 批量接口要数组；单条可改用 --fields，批量用 --records。';
|
return '参数形状不符 —— 批量接口要数组；单条可改用 --fields，批量用 --records。';
|
||||||
|
if (status === 429)
|
||||||
|
return '触发限流 —— 稍等后重试；批量操作请降低并发或分批执行。';
|
||||||
|
if (/timeout|ETIMEDOUT|ECONNRESET/i.test(m))
|
||||||
|
return '请求超时/连接中断 —— 先用只读命令(如 table list)确认服务可达,再重试原操作。';
|
||||||
if (/VALIDATION|Validation error/i.test(m) && status === 400)
|
if (/VALIDATION|Validation error/i.test(m) && status === 400)
|
||||||
return '请求校验失败 —— 用 --json 查看完整错误；对照 `<cmd> --help` 的参数示例。';
|
return '请求校验失败 —— 用 --json 查看完整错误；对照 `<cmd> --help` 的参数示例。';
|
||||||
return undefined;
|
return undefined;
|
||||||
|
|||||||
Reference in New Issue
Block a user