feat(cli): base/space 命令 + 执行守则 + eval harness 入仓 + 多轮 eval (0.1.9)
- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖) - SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引 (snapshot 无恢复 API,用 deactivate+节点改回) - hint 补 429 限流与超时模式 - evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/ EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分 - 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用 - 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次; 多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话) - 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Attempts-to-success distribution for iteration-5 (N=3 variance rerun).
|
||||
|
||||
For every CLI verb session across all runs: which attempt number succeeded
|
||||
(1 / 2 / 3+ / never)? The user-facing goal is "1-2 attempts per call".
|
||||
"""
|
||||
import json, glob, os, sys
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from analyze_trajectories import load_run, classify_output, cli_verb
|
||||
|
||||
def sessions_for(pi):
|
||||
calls, meta = load_run(pi)
|
||||
cli = []
|
||||
for tool, cmd, out in calls:
|
||||
if tool != 'bash':
|
||||
continue
|
||||
cli.append((cli_verb(cmd), cmd, classify_output(out, cmd)))
|
||||
# verb -> attempts used until success (per contiguous session)
|
||||
stats = defaultdict(lambda: {'sess': 0, 'a1': 0, 'a2': 0, 'a3p': 0, 'never': 0})
|
||||
cur = defaultdict(lambda: {'fails': 0, 'done': False})
|
||||
for verb, cmd, sig in cli:
|
||||
if not verb:
|
||||
continue
|
||||
st = cur[verb]
|
||||
if sig is None:
|
||||
if not st['done']:
|
||||
stats[verb]['sess'] += 1
|
||||
n = st['fails'] + 1
|
||||
if n == 1: stats[verb]['a1'] += 1
|
||||
elif n == 2: stats[verb]['a2'] += 1
|
||||
else: stats[verb]['a3p'] += 1
|
||||
st['done'] = True; st['fails'] = 0
|
||||
else:
|
||||
if not st['done']:
|
||||
st['fails'] += 1
|
||||
for verb, st in cur.items():
|
||||
if not st['done'] and st['fails']:
|
||||
stats[verb]['sess'] += 1; stats[verb]['never'] += 1
|
||||
return stats
|
||||
|
||||
def main():
|
||||
roots = sys.argv[1:] or ['/root/eval-rerun/iteration-5']
|
||||
cfg_glob = os.environ.get('EVAL_CFG', 'with_skill')
|
||||
tot = {'sess': 0, 'a1': 0, 'a2': 0, 'a3p': 0, 'never': 0}
|
||||
zero = dict(tot)
|
||||
per_verb = defaultdict(lambda: dict(zero))
|
||||
npass = nrun = 0
|
||||
files = sorted(f for r in roots for f in glob.glob(f'{r}/eval-*/{cfg_glob}/run-1/pi.jsonl'))
|
||||
for pi in files:
|
||||
nrun += 1
|
||||
gp = os.path.join(os.path.dirname(pi), 'grading.json')
|
||||
if os.path.exists(gp):
|
||||
try:
|
||||
g = json.load(open(gp)); exps = g.get('expectations') or []
|
||||
if exps and all(x.get('passed') for x in exps):
|
||||
npass += 1
|
||||
except Exception:
|
||||
pass
|
||||
for verb, st in sessions_for(pi).items():
|
||||
for k in tot:
|
||||
per_verb[verb][k] += st[k]
|
||||
tot[k] += st[k]
|
||||
print(f"runs={nrun} eval-pass={npass}/{nrun}")
|
||||
s = tot['sess']
|
||||
if s:
|
||||
print(f"\nverb sessions: {s}")
|
||||
print(f" attempt 1: {tot['a1']:4} ({tot['a1']/s*100:.1f}%)")
|
||||
print(f" attempt 2: {tot['a2']:4} ({tot['a2']/s*100:.1f}%)")
|
||||
print(f" attempt 3+: {tot['a3p']:4} ({tot['a3p']/s*100:.1f}%)")
|
||||
print(f" never ok: {tot['never']:4} ({tot['never']/s*100:.1f}%)")
|
||||
print(f" ≤2 attempts: {(tot['a1']+tot['a2'])/s*100:.1f}%")
|
||||
print(f"\nper-verb (only verbs with a retry or never):")
|
||||
for verb, st in sorted(per_verb.items(), key=lambda kv: -(kv[1]['a2']+kv[1]['a3p']+kv[1]['never'])):
|
||||
if st['a2'] or st['a3p'] or st['never']:
|
||||
print(f" {verb:24} sess={st['sess']:3} a1={st['a1']:3} a2={st['a2']} a3+={st['a3p']} never={st['never']}")
|
||||
print(f"\nall verbs:")
|
||||
for verb, st in sorted(per_verb.items(), key=lambda kv: -kv[1]['sess']):
|
||||
print(f" {verb:24} sess={st['sess']:3} a1={st['a1']:3} a2={st['a2']} a3+={st['a3p']} never={st['never']}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user