From c9694511a143e4dd91b2aa4818c3253baf0a1393 Mon Sep 17 00:00:00 2001 From: yeuimu <2197651308@qq.com> Date: Wed, 30 Sep 2026 08:12:16 +0000 Subject: [PATCH] =?UTF-8?q?feat(cli):=20base/space=20=E5=91=BD=E4=BB=A4=20?= =?UTF-8?q?+=20=E6=89=A7=E8=A1=8C=E5=AE=88=E5=88=99=20+=20eval=20harness?= =?UTF-8?q?=20=E5=85=A5=E4=BB=93=20+=20=E5=A4=9A=E8=BD=AE=20eval=20(0.1.9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新命令: space list/create、base create/delete(实测通过,冒烟覆盖) - SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引 (snapshot 无恢复 API,用 deactivate+节点改回) - hint 补 429 限流与超时模式 - evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/ EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分 - 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用 - 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次; 多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话) - 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403) --- evals/README.md | 25 ++ .../analyze_trajectories.cpython-313.pyc | Bin 0 -> 14881 bytes evals/analyze_attempts.py | 83 ++++ evals/analyze_trajectories.py | 222 +++++++++++ evals/grade_evals4.py | 376 ++++++++++++++++++ evals/grade_multiturn.py | 111 ++++++ evals/grade_plugin_evals.py | 150 +++++++ evals/plugin-evals.json | 41 ++ evals/run_evals4.py | 156 ++++++++ evals/run_multiturn.py | 83 ++++ evals/run_plugin_evals.py | 113 ++++++ evals/teable-evals.json | 86 ++++ package.json | 2 +- scripts/smoke-commands.sh | 51 ++- .../dynamic-plugins/deployment.md | 6 + skills/inkreach-teable/SKILL.md | 7 + .../inkreach-teable/guides/cli-reference.md | 9 + src/commands/resources.ts | 79 +++- src/lib/client.ts | 12 + src/lib/ctx.ts | 4 + 20 files changed, 1607 insertions(+), 9 deletions(-) create mode 100644 evals/README.md create mode 100644 evals/__pycache__/analyze_trajectories.cpython-313.pyc create mode 100644 evals/analyze_attempts.py create mode 100644 evals/analyze_trajectories.py create mode 100644 evals/grade_evals4.py create mode 100644 evals/grade_multiturn.py create mode 100644 evals/grade_plugin_evals.py create mode 100644 evals/plugin-evals.json create mode 100644 evals/run_evals4.py create mode 100644 evals/run_multiturn.py create mode 100644 evals/run_plugin_evals.py create mode 100644 evals/teable-evals.json diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..4201168 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,25 @@ +# Skill/CLI eval harness + +14 题 teable 操作 eval + 6 题 plugin eval + 逐次尝试统计。单源真值在本目录, +运行产物默认落 `/root/eval-rerun`(可用 `EVAL_ROOT` 覆盖)。 + +## 目录 +- `teable-evals.json` / `plugin-evals.json` — 题目(prompt 模板,支持 {TAG}/{WFID}/{TBL5}/{ORD}/{TBL}) +- `run_evals4.py` — 启动器。支持配置: `with_skill=:`, `old_skill=...`, `auto_skill=NONE:`(触发模式:不给技能路径,靠用户级技能自动发现);`EVAL_MODEL` 换模型 +- `grade_evals4.py` / `grade_plugin_evals.py` — 判分(按最终状态优先)。重判历史轮次须冻结该轮 env(`EVAL_ENV`) +- `run_multiturn.py` + `grade_multiturn.py` — 两轮修正式对话 eval(pi `--session-id` 续接;判分认 recordCondition 或 recordUpdated+门控两种正确形状) +- `analyze_attempts.py ROOT...` — 尝试次数分布(第几次成功/never;`EVAL_CFG` 选配置目录) +- `analyze_trajectories.py` — 轨迹解析公用库(失败签名分类器) + +## 一次完整 A/B +```bash +EVAL_ROOT=/root/eval-rerun python3 evals/run_evals4.py iteration-X A1 \ + "with_skill=/path/skill-new:/path/bin,old_skill=/path/skill-old:/path/bin-old" +python3 evals/grade_evals4.py iteration-X A1 +python3 evals/analyze_attempts.py /root/eval-rerun/iteration-X +``` + +## 环境要求 +- pi(opencode-go 凭据)+ 隔离 HOME(无用户级技能,除 auto_skill 模式用带技能副本) +- 每轮独立沙箱 base;判分后清理。**一切建库仅在 agent-playground 专用空间**。 +- eval-12 需低权限 PAT(sandbox-env.json 的 LOWPERM_TOKEN) diff --git a/evals/__pycache__/analyze_trajectories.cpython-313.pyc b/evals/__pycache__/analyze_trajectories.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a0a3fc9ea72161baa3d795c0b6b076fc2548e9f GIT binary patch literal 14881 zcmch8dr%umnrD?%0wD>|gQuZQ@vx0-9tQlt_|+KWH(*ngjSbzjKmugs60Q<{Xgod4 z#NHlE&#XoF^jPlmo@MVw7;o?1Fn1GobkD{Fb1`$_iM_axS{}pHa5Eiq7Z>4(`@^r; z+1|T|yUw--Jm%se-%j9Ktwu!;>>?ePHaYq%y{5SHUJ%(iD zrHN&jj~RiH8G#jaCs~=LZ=I~8Z@sLiZ%*du+aMe8)}PEcktt{LOg$rTXIRbbO_l9^ZN6JoR(gOaTyt=Kd*w>nIy_HBf1ft(>n1Ju+}5HE_#Ln ze)&4@l3c#)L66K2`(=LITdRzDeZE?sYmoO`aryXw?2?or**hLk>P)87E?My?C{}k- z@k^_6Yk1i+9yJTVkN5szFePe^>g@AuUXxqLoe@m!wpNJAc_#^m+{(7M78ySzS+ zo3BUJfb2zV$badQOL8lGwSOWoJ`t!5c&-KbQO|_zRRZ22h5y~;_t5$lg&%Y&qm;M_ zbthy=;Uz!sx#kUkhTEe!YtS)5PnJDCR{%U;^#(@y5!vPTN+Xne9!;P<59wDIZB1!0 z@q9yFig#rK+(1Oto?!?lz;B}rPskq7oshu)Hs0+Ys+B!1H}99-9@&Zdjdi?hWJLCi zpjCIOg5n*KT;P)k53+`!hP@*w-Bia9du1h1>k0%sm&OCUGBE@ZDZGs8bVvfDq&c+N zcIr5P#Ul?Yrl({`J0G&D+3gv2P51(C?@++$n8-!)yZy@6zTe%s^}9Ra-~H)#-U*sX zf=8D9a{pk^*gx1#qAfw|S(nf2b_KkCiKn?iTT=FrV*m|LNNzQwy|b&mvq!bGw{>-P zcDEl9h4#a0R(sF!j`qV{A~Gf<*F<2{FMER?cQCtu5Mmniy4@ZLjafUqiUMI{a4%1I zWfUlykj5ncRSBazPFlW^Y|xD)9yrbc@;_k5>_f&hy=8{TaQc7Vy`>ZB)%p*H@p&*>qa*e zR(2nE-#7*Bat4b;NtqZQ$4Fz~eBMEsjD6txxF={;#$8}r8o$IRMZ(?)9(cFf>%Jkm zE_rT189g9#)h~|?`}|iCg*uOnc;p_`MlZ==-;kinH_#8y4KP^c97@h_Iz2grdi-9V zrf4NH)ijx5-1r^=Zi(z(q)0lEpqQRxB)zE9%DCQQn7(9rhUreHn@j~xaU!E3N~#{E zEi&C2R;FkGw~}!ObF3fjaw6BAnUJ4(TkQ7hkV)?zDfk<*ZIRlR10 zN$VF-5$$HL)Y4*tVQp;}Y4sc_5BX$pjH2V2D#pwJ^;lJ^c>4;LKEYvPU&sas`G)Rg*IA^9Fog$)l(Su;Cq7 zGmo?$KQ++Zu4X76mrTn9Tyj7m5au+f905gDKXl2hW_pzY!i~y}0)$kZPg2NW^5jw5 z>Sez_Q1AAP`~250c>q}TYj~_3@_FkC%q#WDWnch!pI8~Z0LJm_YVMHFg=su|eIOwc zxft}z@l*a7!6frP%(+XsjxTcx!j;qh$)ii=+z=OXOtnlN{!M1i_Zn|Cf7l!?J@SZK zbaccV9Voh#W54O2_CLufo6;?1=icm{?)~7xl+Nuxwig%5OkWx?PpUOy8)7B-cVk@PIPI{QC#!NUc|=i2CwQ5d3Tg%hf51O1!|Gug@l$pnm}HjC zkj|aMi{|o}xqKPRNRESdXKU?`gUEL!i^s-g?COt5s51%1s!j6BigNC9+; zy1A=x6>Y0l@ip4uSTspyk?nS9WmevhSPqeKU&^c%Q^u@lhAC?0m>ntQ=9!LvpyObp zSTz6IpWl*25LsCnTjV6fK>{Z#p12z5ds6AYo4JAVzLZi zdddOSL{i((#BsN3UsJ*(xn)cSOj1~TFwV$4mjZxa8LJ|rvZhr?MdZ|k-^&1kN_6mJ zAPQF9I!q@vZL+a{0JqSC4$NZ=D519I5io%@6}ZBDY`~ChPY8KLw_6VZ>Sdw^l(|m1 z4AF-=RaduNWv?kjG<*_rvi6O5V8V>c2hTv?=|fX^N8HL!wLZ&WvU6^>Pq&9Qf6y_-E}1P+>*lz5 z^U~%mp@vZZ%)z*&aw$6}YTq2s-n?WjxMi9#g&V^Cw-3%9id*ZaT9+Jow|36#3>QS~ zaYxnE;U%l>mSM&a8vQ7D(Yh(ph!V*ZFKuwio;TIDoLd|&ztgjrTM^5xSSr|X%RA$Z zmGykq5HIVA7xc{Nmu)5C);r3gtukh-Tq-CG>6a|Fo2RBvh0DVyZrA>0!5sH#>6|jZ zW4>+fdbGIpiRBQu4AsQ5H!fMMp~4S3!e?TZjepWXQtO{&moIIsm^vIP{!Z8PTE@2V zSsi1sP3?_x1%LO-$ykeDGHh1W*J@7 zi2Vh70Ab>JgSE`ASj!SCuB`YNun|Vktvk>30A}_NS;?=1HnjwTUDXF$-o>Wi#^%j8 z_TG@juk|b2kTl{AZ0i@c;lp{*xyEoKCHh>o!PsEE1%wh^LyAVl( z7_2USTgsS0sSd17BAWyW2$h=83V3Sdh7=bGlBNU&%&&MaYPs4!&p)?#SF4vbnPPx?s_^Y zq!x8e0bBrAQ7>eXw$?LbKqqCoph)r&GPOPnCQ?)T!YXqb^bCTz;1lHTxF7c7v+swZMq-*KBbWTCJHc07$Jv4 ztsN;lA3gYCVhV_Q%+uV?Rdb3%U+J7`K_qQXL4E0f*!>LxL+u>Wb#zb&pi_p(n1`I)fR}uf1Y|W;S-(P>BP$ABN)pp5yJVkNHTmI0 zlLv^m5&J6tkRrc>!a-v_^>r%sL4DmNw>*fnr~iP5l9J!Ur|cqWZUjyvUA9%d5B7x~ zjzQH3bAj&2<+qWKtvlJq)7>#eB&SJkMY?>51Z|{*9tIP3`yP)((Nckd2}>sdZhF|J zAnP4b*((YGA2Qg9Ct*EgYHnITGKsvDOlmwV+c$)F$L*EhJUQ9^rL}O`mKPE~+OTYO z#2hX6#JHm+Zf!wgEN|~UPdsmL+_rbgmLD3Kd27j07|xnGNH&QDMN3xubBD>CG1>O4 zfU)OIp7`~Kvgw>DF4RQ!^SeGcuml4@i0ASV;X(hs{<%$yo0?;rnwPAO(2*JQqO~Gs zt%x)|*mG~s?1gw`^SnJ)x$m)c|F0~0$@+*b=e!jjisx*Z(l1-`LISZ~2ryf6Lx$-S zibiLYJr z=hY-49=FGR&{+pztvp2Z0BYu?6+zMoLeGurIJBwnq?<-3r&>rm%0Pm;YLGnOL(QVC zz>{`e&7s(YZ3a86*4;35t{r>lp^dZ97W0hZYhivdE%Dg z$@XPaeyrd?+;niV?bjPBlOUIEojwn5q&a7LFL5xA-8#5r&7UzxOKalR+BsXy+K4wz z3U|h=+ZL^jNSbS$8~Neh$JT?S%E-*#NO{~*xs+Etb24I%=he)yvAhPnmnt@gbfLX7 zxl0?%5ZV2a`FS>zSNY7!oj@lb?tzMDFKoH6J{MzaS0kiJ$ce<*TahA;SHN598Tmgg1$+w!I$=?)UW{J*kWBE zN%QpS`^do3M@*Su5|?&}Kvu;y&w57#|45CLskuUGr9B)g0G`Mr83e%nKSB%ew)PQU zKbRP17_f5qKxh27dyf;k1G)~t64|jTIw`9?wH4!uzpy=# zgBX@QQl_s`<5sY1a$c3{ST{ATtdJ+<=Q2WpP$(49cd@X6zDtttQsFK7y)pS+mVEQc z^m3sB`ISv}$+W_anDQ3$2{+)6po`|dO(|}xFnVU8jP!V&F|?$W-1G-aW@}Wmg_$ha z!saek*dpbm@naV9NL$z*Rdevvk{+REAx)$G?jx24eUNS>>6+wF$B8;rD#+VP>u6uw z=ogg`@@iJ{`kYB}seY#DCF)ud@)NVE{0df;uT5_sm{bSa^(0N}CD{xL+tEu?4%3vc zQRbPBf4^2QG$Ut~R-}#pij0JY6*_6D>1DTRed?>xVp<(3ml~T`I^mH|jqEWT>X3b; zWR+}^U9_k3-vOJX_kShEK(vZ>$hndDQ;RbHjMla%TI;~FOlKe#=ENiBcGqk1mA58;O-|qr3=r<9TpW;$ zUCK+7_ioWaXJWHx74|?+ThcTmPopC2Mb19aj?qt{8YF@CSEGW~`afax_iMD@ST;5K z!hy6+Mm`<)5S`x*TE8_7&}d2pV!0(7!pFnBYs#397yh?{*l8VKrbwrnw zm|dLCQwQullHT!Q;w|Q*r*Ps(j<>v#IJSWij_)BW z3HJC@^Q{Sakv?ery0XD5WJJ=?9!KPvh&SmeUT#R^X&|7Ajm)H((9X`CKBELVpW2nNz17 zbVDo=3wP+CCk2WAD4bM?k|m&*#4JippQMJ7qN7M?l>)VJ1f8r8yN~Lbp411BFp_0O zHi7_pJ<0JU>zkMw>m+v4aZk=I#LuD(?Kk4*5T|PlM#8cpIng5Y3Jgb375%4Nl}LqY zZJrlPQ2T;dh0$(5V^YxVz$UcRJa`+23|lNi6H1#Rj={_Z@^fZG6WQP;v zh$0Yj>;saa)t`6i7wG^#rPn9A3>D?QhMNMxZiwx6V5eVpI)ZU^yZ{9J-d|sD68}h z&h&pS`|_U>R!@`QUlVqSY7sZc%7Fl7_1}=Z6s!47_j_-*hg&1;zq8$P&A9IOekI>U z36(vsvb9e)Bl;9ZCX>*$6*Z%JrE~Mt&91Tsf(OWjG_kv#7mgq8Z0$iz;H#4^HgYYs zo~Iu6T6!s$`f+f>B6ZEx`Sb{Bi8$-2-Ubb%T+353a4Opu)V;$)X&QriUfQp+jX@pH z2WwkbeTc74YhVbJQV;17b$bzq*qeO6aDwSN zS5+hPlvcyClkk?o;a88t-Zd^ijyUsF1NHGymzGLyAj{VWM&W#Xqw`FyUIQ{oXWr}X zy`Q7`q|w_XnZf>W}~A;wSaUkG5EwcrxvVFo)ROTlS$^zSZwQik2ks! zgovUf{B=Jdu>c7a63mf6MiAs`Yis!+$K$mMreT671cmtpL6IA&M0Ntb^0O7UE{T71 zH&SHU2;ml2*mPr8(k`$i9Ew=tnE!%bvbzhqg@&IR7KVSC{n`7JOH|X3d9Fh_b_7kh zKb~-+?hl%iccvkAD9mo;QO9eoe{6ryLhq7auXSTI_D_(_Dbd~b!}a9SGVUz%fG&Yv zBTwCQ#3f&NagRK0L}9s(1m7XScM$}Qd}0aNAIt&+^kUF{@89E)2su1YNb^Bm3-8>3 z%OUjQu*!}2{Dbl+q2NaV_*3XPRpL-P)9sbXm0>jlch%v}lv_v{j@%8A_mDajOE_-p0Xqe<2B!Pzn1Pf&N#2NW(B={x*oF(Dn;9Dvu`>!gr zJF0Hni~8gmDP3k%-N-myYE*UvOVG7(MPG|&$$_Q<|33vU0$}St3CB1` zbXxubg~?i?e1b)El36lmPaa;*IXI-I3cPcm22hvxD)C4RZ(L zwnM0vUp#eWIoEdc>h#r{?@zxU9$d^VkL8v>(;02W%SEMkvp&iC^QwsaXVvkdZK2ks zlG3|7Z|{t3jF)U*EZH@8D0KKYg(b`7Rk5nJ&)o5=kICr~sapq#A6zvx@%zUlQ*;h4;!8GN3%;ha}Sy9vG$J`b~Xu?vqBXnr$Wax4@KP-n$k@CplSzDxe zwqaHT<>PZ_<^uDrbEEUF1;c`C{`w>K(WdC>Gmna+-9ohIT(tLmRJahm&=>6=in=|~ zzTxQbSk&a3Y$Nn5|LDZ8vu#r>J&>F6Ui+IZqo`J>iHBfmHiJ=go_K=k~DXkUNy9e1qH6CECjUKovz zdZQP|qCO>dF%X@&67^n)w?$-`7eI3I`@?u<@%;^I?fNcnB9eM(uf=7kM2qY)0rpOv$X zkm;dKIe(`=yL;8wguW%ydhk3CwKbTR9nbFE?2eB zRELj7+Gh2$_Q}b@sJ7#HKv>c3C4lb|@n}3@1sPa+cLe3LQ*GoN9 zT%G8Hg{j&)TlIsj54V0=^Q3Zj%vcsP7EL*(+@ZGcCNe@Sm>moih3(5kv)&OmXt)u;fyP88pPoYRhw*{jKjU6 zc=lGD&lDElx;As|`@!j>5K%cfeQphTW{%Bdhutx2C2p=+vZsbWI3Bh|OSZ*|Yhsp~ z501~e<}yFMINv%y_E~4F?!>1TiE8}KiG;$&F1TZ-+<$X|RP=q&g|UA3i*5VjWrq+( zTt8zUu(56XVr7S7mP6m|dYQ=-RY1r&_LmmM>cBZRde4P~P`7eX(w+^1IP91JQx8XAIM!>(t@ZrR&rE zo_zJ|9{c3S$`vRyl7s9K-E)Sy#vawZWS9VZO!tamF0sc+l|h}GynL*Oyd1;2zh#*0 z5#37zQ`E(xmsZCs%j<*H^qAYB4OT*F&6cU&>Am4iUzp1xmuHI~+_-mRXI4Qwlw9d?4d3c`8&3W7W&d-YGO$*hJ8Xooj zV(+8c=(+czuEFS#FXp-weNT$o{f{l<1YQTjL;uP&!$EHwmaX|=BgR3GV>MSqh!!}X zSho`0uY5t~@87-hvrJJrE*sj4)x09iZ(H&IeqmxZz0ZF2qLXF#HukGm8<_l-uj2bW z3R!^vR$0Jf{8pyn)4GY1B+B^nr@W`g`V66GJ00In9Y#$Q)d^N`=?!+x3` zFB3~ZHPOEwId#0VU7&wrFpj^mqsE79CE09;YUnzNzb;AKRjDH6PLZI81jJ;a76e^) z2}{IU0RyK=M#|_^jr%V7-4i~~0eKl2WN)raBY=!p_Bm%_x!)HsY}q2SG0trKuT13& zi!nzxX?kX7%-J`$etT<3xYaw;8@?FNcg9UyCo{jqr9IQPO;JZr^lb0q*$c6=7vjdg zNy95Hm$kmA(6f79m`d2f7cCsS{RM6<<-TYsU~Mn9pJLf<&-!%?XPFAd^~Jy9jQ`@u LGd;s?psfBMXb!f- literal 0 HcmV?d00001 diff --git a/evals/analyze_attempts.py b/evals/analyze_attempts.py new file mode 100644 index 0000000..5a5d21f --- /dev/null +++ b/evals/analyze_attempts.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Attempts-to-success distribution for iteration-5 (N=3 variance rerun). + +For every CLI verb session across all runs: which attempt number succeeded +(1 / 2 / 3+ / never)? The user-facing goal is "1-2 attempts per call". +""" +import json, glob, os, sys +from pathlib import Path +from collections import defaultdict +sys.path.insert(0, str(Path(__file__).parent)) +from analyze_trajectories import load_run, classify_output, cli_verb + +def sessions_for(pi): + calls, meta = load_run(pi) + cli = [] + for tool, cmd, out in calls: + if tool != 'bash': + continue + cli.append((cli_verb(cmd), cmd, classify_output(out, cmd))) + # verb -> attempts used until success (per contiguous session) + stats = defaultdict(lambda: {'sess': 0, 'a1': 0, 'a2': 0, 'a3p': 0, 'never': 0}) + cur = defaultdict(lambda: {'fails': 0, 'done': False}) + for verb, cmd, sig in cli: + if not verb: + continue + st = cur[verb] + if sig is None: + if not st['done']: + stats[verb]['sess'] += 1 + n = st['fails'] + 1 + if n == 1: stats[verb]['a1'] += 1 + elif n == 2: stats[verb]['a2'] += 1 + else: stats[verb]['a3p'] += 1 + st['done'] = True; st['fails'] = 0 + else: + if not st['done']: + st['fails'] += 1 + for verb, st in cur.items(): + if not st['done'] and st['fails']: + stats[verb]['sess'] += 1; stats[verb]['never'] += 1 + return stats + +def main(): + roots = sys.argv[1:] or ['/root/eval-rerun/iteration-5'] + cfg_glob = os.environ.get('EVAL_CFG', 'with_skill') + tot = {'sess': 0, 'a1': 0, 'a2': 0, 'a3p': 0, 'never': 0} + zero = dict(tot) + per_verb = defaultdict(lambda: dict(zero)) + npass = nrun = 0 + files = sorted(f for r in roots for f in glob.glob(f'{r}/eval-*/{cfg_glob}/run-1/pi.jsonl')) + for pi in files: + nrun += 1 + gp = os.path.join(os.path.dirname(pi), 'grading.json') + if os.path.exists(gp): + try: + g = json.load(open(gp)); exps = g.get('expectations') or [] + if exps and all(x.get('passed') for x in exps): + npass += 1 + except Exception: + pass + for verb, st in sessions_for(pi).items(): + for k in tot: + per_verb[verb][k] += st[k] + tot[k] += st[k] + print(f"runs={nrun} eval-pass={npass}/{nrun}") + s = tot['sess'] + if s: + print(f"\nverb sessions: {s}") + print(f" attempt 1: {tot['a1']:4} ({tot['a1']/s*100:.1f}%)") + print(f" attempt 2: {tot['a2']:4} ({tot['a2']/s*100:.1f}%)") + print(f" attempt 3+: {tot['a3p']:4} ({tot['a3p']/s*100:.1f}%)") + print(f" never ok: {tot['never']:4} ({tot['never']/s*100:.1f}%)") + print(f" ≤2 attempts: {(tot['a1']+tot['a2'])/s*100:.1f}%") + print(f"\nper-verb (only verbs with a retry or never):") + for verb, st in sorted(per_verb.items(), key=lambda kv: -(kv[1]['a2']+kv[1]['a3p']+kv[1]['never'])): + if st['a2'] or st['a3p'] or st['never']: + print(f" {verb:24} sess={st['sess']:3} a1={st['a1']:3} a2={st['a2']} a3+={st['a3p']} never={st['never']}") + print(f"\nall verbs:") + for verb, st in sorted(per_verb.items(), key=lambda kv: -kv[1]['sess']): + print(f" {verb:24} sess={st['sess']:3} a1={st['a1']:3} a2={st['a2']} a3+={st['a3p']} never={st['never']}") + +if __name__ == '__main__': + main() diff --git a/evals/analyze_trajectories.py b/evals/analyze_trajectories.py new file mode 100644 index 0000000..aeff2d5 --- /dev/null +++ b/evals/analyze_trajectories.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Trajectory analyzer for pi-skill-lab eval transcripts. + +Parses pi.jsonl transcripts, reconstructs per-run tool-call sequences, +detects failed / retried commands (output-text heuristics — pi's bash tool +returns no exit codes), and correlates with grading.json outcomes. + +Outputs: + 1. per-run summary (effort + failure counts + doc-read order) + 2. aggregated failure signatures per config + 3. first-attempt success rate for the core CLI verbs +""" +import json, glob, os, re, sys +from collections import defaultdict + +# --- failure heuristics on tool OUTPUT text (stdout only in pi) ------------- +# Strict failure patterns: an output line is a FAILURE only when it looks like +# an actual error line — NOT when the word "failed"/"error" appears in normal +# status output (e.g. `runs list` printing status=failed, or "error": null). +FAIL_LINES = [ + r'^\s*[✗✘❌]', r'^\s*error\b', r'\bError:', r'Validation error', + r'command not found', r'ENOENT', r'ECONNREFUSED', r'ETIMEDOUT', + r'unauthorized', r'\bforbidden\b', r'Missing required', + r'unknown (option|command)', r'is required', + r'HTTP_?[45]\d\d', r'Cannot (GET|POST|PUT|PATCH|DELETE)\b', + r'"status":\s*[45]\d\d', r'Invalid (option|value|input|RecordId|List)', + r'Unsupported library type', r'spawn npm ENOENT', +] +FAIL_RE = re.compile('|'.join(FAIL_LINES)) +# outputs that legitimately contain these words but are NOT failures +OK_RE = re.compile(r'^(id|name|baseId|workflowId|triggerType|config|next|note|\s*$)', re.M) + +def classify_output(text, cmd=''): + """Return failure signature (short str) or None if output looks successful.""" + if not text: + return None + # `--help` legitimately prints "Usage:" — not a failure + is_help = '--help' in cmd + head = text[:400] + if is_help and re.match(r'\s*(Usage:|inkreach-teable \[options\])', text): + return None + for ln in text.splitlines(): + ln = ln.strip() + if ln and FAIL_RE.search(ln): + if is_help and ln.startswith('Usage:'): + continue + return ln[:140] + return None + +def cmd_of(args): + if not isinstance(args, dict): + return '' + return args.get('command') or args.get('path') or '' + +def load_run(pi_path): + """Return ordered tool calls [(tool, cmd, out_head)], plus meta.""" + calls, meta = [], {'turns': 0, 'settled': False} + starts = {} + for line in open(pi_path): + try: + e = json.loads(line) + except Exception: + continue + t = e.get('type') + if t == 'tool_execution_start': + starts[e['toolCallId']] = e + elif t == 'tool_execution_end': + s = starts.get(e['toolCallId'], {}) + args = s.get('args', {}) + r = e.get('result') or {} + c = r.get('content') + txt = '' + if isinstance(c, list): + txt = '\n'.join(x.get('text', '') for x in c if isinstance(x, dict)) + elif isinstance(c, str): + txt = c + calls.append((e['toolName'], cmd_of(args), txt[:600])) + elif t == 'turn_start': + meta['turns'] += 1 + elif t == 'agent_settled': + meta['settled'] = True + return calls, meta + +def cli_verb(cmd): + m = re.search(r'(?:^|\s|&&|;|npx\s+(?:git+\S+\s+)?)inkreach-teable\s+([a-z-]+)(?:\s+([a-z-]+))?', cmd) + if not m: + return None + return '.'.join(x for x in m.groups() if x) + +def analyze(pi_path): + calls, meta = load_run(pi_path) + run = { + 'path': pi_path, + 'total': len(calls), + 'bash': sum(1 for t, c, o in calls if t == 'bash'), + 'reads': sum(1 for t, c, o in calls if t == 'read'), + 'turns': meta['turns'], + 'settled': meta['settled'], + 'cli': [], # (verb, cmd, fail_sig_or_None) + 'other_fail': [], # non-CLI failed bash (npm, curl, node...) + 'docs': [], # skill files read, in order + } + seen_verbs = {} + for tool, cmd, out in calls: + if tool == 'read': + if '/skills/' in cmd and cmd.endswith('.md'): + run['docs'].append(cmd.split('/skills/')[-1]) + continue + if tool != 'bash': + continue + sig = classify_output(out, cmd) + v = cli_verb(cmd) + if v: + run['cli'].append((v, cmd, sig)) + elif sig: + run['other_fail'].append((cmd[:100], sig)) + return run + +def attempts_to_success(cli_calls): + """Per verb: list of (#failed attempts before first success, succeeded?).""" + stats = defaultdict(lambda: {'fails_before': [], 'never': 0, 'first_try': 0, 'total_sessions': 0}) + cur = defaultdict(lambda: {'fails': 0, 'succeeded': False}) + for verb, cmd, sig in cli_calls: + st = cur[verb] + if sig is None: + if not st['succeeded']: + stats[verb]['total_sessions'] += 1 + stats[verb]['fails_before'].append(st['fails']) + if st['fails'] == 0: + stats[verb]['first_try'] += 1 + st['succeeded'] = True + st['fails'] = 0 + else: + if not st['succeeded']: + st['fails'] += 1 + # verbs that never succeeded + for verb, st in cur.items(): + if not st['succeeded'] and st['fails']: + stats[verb]['total_sessions'] += 1 + stats[verb]['never'] += 1 + return stats + +def main(): + rows = [] + for ws in ['inkreach-teable-workspace', 'plugin-workspace']: + for pi in sorted(glob.glob(f'/root/pi-skill-lab/{ws}/iteration-*/eval-*/*/run-*/pi.jsonl')): + parts = pi.split('/') + run = analyze(pi) + gp = os.path.join(os.path.dirname(pi), 'grading.json') + passed = None + if os.path.exists(gp): + try: + g = json.load(open(gp)) + exps = g.get('expectations') or [] + if exps: + passed = all(x.get('passed') for x in exps) + else: + passed = g.get('passed') + except Exception: + pass + rows.append({ + 'ws': ws.replace('-workspace',''), 'iter': parts[4], + 'eval': parts[5], 'cfg': parts[6], 'run': parts[7].replace('run-',''), + 'passed': passed, **{k: run[k] for k in ('total','bash','reads','turns','settled','cli','other_fail','docs')}, + }) + # ---- 1. per-run summary ---- + print('=' * 110) + print('PER-RUN TRAJECTORY SUMMARY') + print('=' * 110) + hdr = f"{'eval':38} {'cfg':12} {'ok':4} {'tcs':4} {'rd':3} {'trn':4} {'cli':4} {'f':3} {'retry':5} docs" + print(hdr) + for r in rows: + fails = sum(1 for v,c,s in r['cli'] if s) + retries = sum(1 for v,c,s in r['cli'] if s) # failed call == at least one retry if task later passed + cli_ok = 'Y' if r['cli'] else '-' + print(f"{r['eval'][:38]:38} {r['cfg']:12} {str(r['passed'])[:3]:4} {r['total']:4} {r['reads']:3} {r['turns']:4} " + f"{len(r['cli']):4} {fails:3} {retries:5} {len(r['docs'])}d {'>' if not r['settled'] else ''}") + # ---- 2. failure signatures ---- + print() + print('=' * 110) + print('FAILURE SIGNATURES (failed CLI / other-bash calls, per config)') + print('=' * 110) + sigs = defaultdict(list) + for r in rows: + for verb, cmd, sig in r['cli']: + if sig: + sigs[(r['cfg'], verb, sig[:70])].append(r['eval'][:30]) + for cmd, sig in r['other_fail']: + sigs[(r['cfg'], 'bash:'+cmd[:40], sig[:70])].append(r['eval'][:30]) + for (cfg, verb, sig), evals in sorted(sigs.items()): + print(f"[{cfg:12}] {verb:32} n={len(evals):2} {sig}") + # ---- 3. first-attempt success per verb, per config ---- + print() + print('=' * 110) + print('ATTEMPTS-TO-SUCCESS PER CLI VERB (per config)') + print('=' * 110) + agg = defaultdict(lambda: defaultdict(lambda: {'sess':0,'first':0,'never':0,'retry_hist':[]})) + for r in rows: + st = attempts_to_success(r['cli']) + for verb, s in st.items(): + a = agg[r['cfg']][verb] + a['sess'] += s['total_sessions']; a['first'] += s['first_try']; a['never'] += s['never'] + a['retry_hist'] += [x for x in s['fails_before'] if x] + for cfg in agg: + print(f'--- {cfg} ---') + for verb, a in sorted(agg[cfg].items(), key=lambda kv: -kv[1]['sess']): + ft = f"{a['first']}/{a['sess']}" if a['sess'] else '-' + print(f" {verb:34} sessions={a['sess']:3} first-try={ft:7} never-ok={a['never']:2} retry-dist={sorted(set(a['retry_hist'])) if a['retry_hist'] else []}") + # ---- 4. failed runs: what happened ---- + print() + print('=' * 110) + print('FAILED/TIMEOUT RUNS — trajectory tail') + print('=' * 110) + for r in rows: + if r['passed'] is False or not r['settled']: + fails = [(v, s) for v, c, s in r['cli'] if s] + [(c[:50], s) for c, s in r['other_fail']] + print(f"\n{r['ws']} {r['iter']}/{r['eval']}/{r['cfg']} passed={r['passed']} settled={r['settled']}") + for v, s in fails[-6:]: + print(f" FAIL {v}: {s}") + +if __name__ == '__main__': + main() diff --git a/evals/grade_evals4.py b/evals/grade_evals4.py new file mode 100644 index 0000000..9cbde8d --- /dev/null +++ b/evals/grade_evals4.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +"""Grade iteration-4 A/B rerun evals. + +Same checks as the original grade_evals.py, adapted to per-config sandboxes: +- each config (with_skill/old_skill) has its own base + order_logs table + ORD record +- CLI = absolute path to the NEW CLI (reads only; version-agnostic) +Usage: python3 grade_evals4.py +""" +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +HARNESS = Path(__file__).parent # evals/ itself +ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun")) +CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable") +env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'sandbox-env.json'))).read_text()) + + +def cli_json(cfg, *args): + e = {**os.environ, + "INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"], + "INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"], + "INKREACH_TEABLE_BASE_ID": cfg["INKREACH_TEABLE_BASE_ID"]} + p = subprocess.run([CLI, *args, "--json"], capture_output=True, text=True, env=e, timeout=60) + try: + return json.loads(p.stdout) + except json.JSONDecodeError: + return None + + +def transcript(run_dir: Path): + cmds, reads = [], [] + for line in (run_dir / "pi.jsonl").read_text(errors="replace").splitlines(): + line = line.strip() + if not line: + continue + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + if ev.get("type") != "tool_execution_start": + continue + if ev.get("toolName") == "bash": + cmds.append(ev.get("args", {}).get("command", "")) + elif ev.get("toolName") == "read": + reads.append(ev.get("args", {}).get("path", "")) + return cmds, reads + + +def answer(run_dir: Path) -> str: + fa = run_dir / "final_answer.md" + af = run_dir / "assistant_final.md" + return (fa.read_text(errors="replace") if fa.exists() else af.read_text(errors="replace")) + + +def used_official_teable(cmds): + for c in cmds: + # probing for existence (command -v / which) is prudent, not usage + probe = re.search(r"(command -v|which)\s+teable", c) + if probe: + continue + if re.search(r"(^|[^a-z-])teable\s", c) or "npx teable" in c or re.search(r"npm\s+(i|install)\s+teable\b", c): + return c + return None + + +def find_wf(cfg, name_part): + wfs = cli_json(cfg, "workflow", "list") or [] + if not isinstance(wfs, list): + return None # error payload ({"error":...}) — not a workflow list + for w in wfs: + if isinstance(w, dict) and name_part in w.get("name", ""): + return w + return None + + +def wf_nodes(cfg, wf_id): + wf = cli_json(cfg, "workflow", "get", wf_id) or {} + return wf, [n for n in wf.get("nodes", []) if n.get("type") == "trigger"], [n for n in wf.get("nodes", []) if n.get("type") == "action"] + + +def result(text, passed, evidence): + return {"text": text, "passed": bool(passed), "evidence": str(evidence)[:400]} + + +def grade_eval(cfg, ev_name, tag, run_dir, cfg_short="N"): + tbl = cfg["TBL5_ID"] + ord_id = cfg["ORD"] + cmds, reads = transcript(run_dir) + ans = answer(run_dir) + all_cmds = "\n".join(cmds) + out = [] + + if ev_name == "button-click-automation": + # Locate the workflow via the button-field binding (options.workflow) — + # the prompt only fixes the FIELD name, agents may name the workflow + # anything (e.g. 催单自动化A4W), so searching by workflow name misfires. + fields = cli_json(cfg, "field", "list", tbl) or [] + btn = next((f for f in fields if f.get("name") == f"催单按钮{tag}"), None) + wf = None + if btn: + wf_id = ((btn.get("options") or {}).get("workflow") or {}).get("id") + if wf_id: + wf = cli_json(cfg, "workflow", "get", wf_id) + if not wf: + wf = find_wf(cfg, f"催单按钮{tag}") + if not wf: + out.append(result("按钮字段绑定的工作流存在 (催单按钮%s)" % tag, False, + "field list 无该按钮字段或其 options.workflow 无效")) + return out + wf_full, triggers, actions = wf_nodes(cfg, wf["id"]) + out.append(result("工作流已激活 (isActive)", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive"))) + fields = cli_json(cfg, "field", "list", tbl) or [] + btn = next((f for f in fields if f.get("name") == f"催单按钮{tag}"), None) + if not btn: + out.append(result("创建按钮字段 催单按钮%s" % tag, False, "field list 未找到该字段")) + else: + wf_opt = (btn.get("options") or {}).get("workflow") + ok = isinstance(wf_opt, dict) and wf_opt.get("id") == wf["id"] + out.append(result("按钮字段 options.workflow 为对象且指向该工作流", ok, + "options.workflow=%r" % wf_opt)) + out.append(result("触发器为 buttonClick", any(t.get("category") == "buttonClick" for t in triggers), + "trigger categories=%s" % [t.get("category") for t in triggers])) + script = next((a for a in actions if a.get("category") == "script"), None) + out.append(result("script action 读取 triggerInput", script and "triggerInput" in (script.get("config") or {}).get("code", ""), + "code=%r" % ((script or {}).get("config") or {}).get("code", "")[:120])) + fired = any("fire button-click" in c for c in cmds) + runs_any = cli_json(cfg, "runs", "list", wf["id"]) or [] + verified = any(re.search(r"runs (list|get)", c) for c in cmds) or len(runs_any) > 0 + out.append(result("用 fire button-click 实测", fired, [c for c in cmds if "fire" in c][:2])) + out.append(result("执行后查看 runs 结果", verified, [c for c in cmds if "runs" in c][:2])) + runs = cli_json(cfg, "runs", "list", wf["id"]) or [] + ok_run = any(r.get("status") in ("succeeded", "success") for r in runs) + out.append(result("存在 succeeded 的执行记录", ok_run, "statuses=%s" % [r.get("status") for r in runs][:5])) + bad = used_official_teable(cmds) + out.append(result("全程使用 inkreach-teable CLI(未用官方 teable)", bad is None, bad or "ok")) + + elif ev_name == "scheduled-digest": + wf = find_wf(cfg, f"每日统计{tag}") + if not wf: + out.append(result("创建名为 每日统计%s 的工作流" % tag, False, "workflow list 中未找到")) + return out + wf_full, triggers, actions = wf_nodes(cfg, wf["id"]) + out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive"))) + trig = triggers[0] if triggers else {} + timing = (trig.get("config") or {}).get("timing") or {} + ignored = [k for k in ("triggerTime", "weekdays", "daysOfMonth", "starting", "ending", "timezone", "tz") if k in timing] + out.append(result("scheduledTime 触发器", trig.get("category") == "scheduledTime", "category=%s" % trig.get("category"))) + out.append(result("timing 只含生效字段(type+interval,无被忽略字段)", trig.get("category") != "scheduledTime" or (not ignored and "type" in timing and "interval" in timing), + "timing=%r ignored=%s" % (timing, ignored))) + has_stats = any(a.get("category") == "script" and re.search(r"record|count|统计|length", (a.get("config") or {}).get("code", "")) for a in actions) or any( + a.get("category") in ("getRecords",) for a in actions) + out.append(result("有统计逻辑的 action(script 读记录数或 getRecords)", has_stats, + "action categories=%s" % [a.get("category") for a in actions])) + + elif ev_name == "record-condition-email": + wf = find_wf(cfg, f"失败告警{tag}") + if not wf: + out.append(result("创建名为 失败告警%s 的工作流" % tag, False, "workflow list 中未找到")) + return out + wf_full, triggers, actions = wf_nodes(cfg, wf["id"]) + out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive"))) + trig = triggers[0] if triggers else {} + out.append(result("触发器 category 为 recordCondition", trig.get("category") == "recordCondition", + "category=%s" % trig.get("category"))) + c = trig.get("config") or {} + cond_ok = "conditions" in c and "filter" not in c + out.append(result("条件键名为 conditions(无 filter)", cond_ok, "config keys=%s" % list(c.keys()))) + if isinstance(c.get("conditions"), dict): + out.append(result("conditions 形如 {conjunction, filterSet}", "filterSet" in c["conditions"], + "conditions keys=%s" % list(c["conditions"].keys()))) + bad_key = "recordMatchesConditions" in all_cmds + out.append(result("未使用官方键名 recordMatchesConditions", not bad_key, "transcript" if bad_key else "ok")) + email = next((a for a in actions if a.get("category") in ("sendEmail", "email")), None) + ok_email = email and "ops@xingchen.tech" in json.dumps(email.get("config") or {}, ensure_ascii=False) + out.append(result("sendEmail action 收件人 ops@xingchen.tech", ok_email, + "email action config=%s" % json.dumps((email or {}).get("config") or {}, ensure_ascii=False)[:200])) + + elif ev_name == "debug-failed-run": + out.append(result("查询了工作流状态 (workflow get/list)", any(re.search(r"workflow (get|list)", c) for c in cmds), + [c for c in cmds if "workflow" in c][:2])) + out.append(result("查询失败执行 (runs list --status failed 或过滤)", any(re.search(r"runs list", c) for c in cmds), + [c for c in cmds if "runs" in c][:3])) + detail_in_ans = bool(re.search(r"boom|nodeResults|错误详情|执行失败原因|throw new Error", ans)) + out.append(result("查看了具体执行详情 (runs get 或答复引用运行错误详情)", + any(re.search(r"runs get", c) for c in cmds) or detail_in_ans, + [c for c in cmds if "runs get" in c][:2] or ans[:120])) + diagnosed = re.search(r"boom|timeout|超时|抛错|throw", ans, re.I) + out.append(result("最终答复指出脚本抛错原因", bool(diagnosed), ans[:200])) + reactivated = [c for c in cmds if re.search(r"workflow activate", c)] + out.append(result("未盲目重新激活(诊断前不改)", not reactivated, reactivated or "ok")) + + elif ev_name == "select-constraint-write": + inspected = any(re.search(r"field (list|get|visible-choices)", c) for c in cmds) + out.append(result("写入前检查了字段约束 (field list --with-options / field get)", inspected, + [c for c in cmds if "field" in c][:2])) + fields = cli_json(cfg, "field", "list", tbl) or [] + f = next((x for x in fields if x.get("name") == "结果"), {}) + choices = [c.get("name") if isinstance(c, dict) else c for c in ((f.get("options") or {}).get("choices") or [])] + added = "已完成" in choices + recs = (cli_json(cfg, "record", "get", "--table-id", tbl) or {}).get("records") or [] + rec = next((r for r in recs if (r.get("fields") or {}).get("订单号") == ord_id), {}) + val = (rec.get("fields") or {}).get("结果") + claims_success = bool(re.search(r"(✅|已.{0,3}(改|更新|设置|写入).{0,20}已完成|成功.{0,10}改为)", ans)) + if added and val == "已完成": + out.append(result("合法路径:先加选项再写入,记录值=已完成", True, "choices=%s val=%s" % (choices, val))) + elif claims_success and val != "已完成" and not added: + out.append(result("未发生静默失败(声称成功但值未变)", False, + "答复声称已改为「已完成」,但实际 choices=%s 记录值=%r —— 教科书式 mutationApplied=false 静默失败" % (choices, val))) + elif val in (None, "处理中") and re.search(r"不在|无法|无效|不是.{0,6}选项|未写入|没有写入|未添加|无法直接|不能直接", ans): + out.append(result("识别出「已完成」不是有效选项并向用户说明(未静默失败)", True, ans[:250])) + else: + out.append(result("写入了有效选项值或正确说明", val == "已完成", + "choices=%s val=%r claims_success=%s ans=%s" % (choices, val, claims_success, ans[:150]))) + listed = all(x in ans for x in ("成功", "失败", "处理中")) + out.append(result("答复列出了全部可选值(成功/失败/处理中)", listed, ans[:200])) + + elif ev_name == "official-cli-near-miss": + bad = used_official_teable(cmds) + out.append(result("未安装/调用官方 teable CLI 打实例", bad is None, bad or "ok")) + out.append(result("使用了 inkreach-teable CLI", any("inkreach-teable" in c for c in cmds), + [c for c in cmds if "inkreach-teable" in c][:2])) + out.append(result("答复列出了 order_logs 表", "order_logs" in ans, ans[:200])) + + elif ev_name == "webhook-receiver": + wf = find_wf(cfg, f"回单通知{tag}") + if not wf: + out.append(result("创建名为 回单通知%s 的工作流" % tag, False, "workflow list 中未找到")) + return out + wf_full, triggers, actions = wf_nodes(cfg, wf["id"]) + out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive"))) + trig = triggers[0] if triggers else {} + out.append(result("触发器为 webhook", trig.get("category") == "webhook", "category=%s" % trig.get("category"))) + base = cfg["INKREACH_TEABLE_BASE_ID"] + correct = f"/api/webhook/{base}/{wf['id']}" + out.append(result("答复给出正确 URL 形状 /api/webhook/{baseId}/{workflowId}(无 /base/ 段)", correct in ans, + "期望含 %s;答复=%s" % (correct, ans[:180]))) + tested = any(re.search(r"fire webhook", c) for c in cmds) or any("api/webhook" in c and re.search(r"curl|POST", c) for c in cmds) + out.append(result("实际测试了 webhook(fire webhook 或 curl POST)", tested, + [c for c in cmds if "webhook" in c][:3])) + + elif ev_name == "weekday-schedule-trap": + wf = find_wf(cfg, f"工作日盘点{tag}") + if not wf: + out.append(result("创建名为 工作日盘点%s 的工作流" % tag, False, "workflow list 中未找到")) + return out + wf_full, triggers, actions = wf_nodes(cfg, wf["id"]) + out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive"))) + trig = triggers[0] if triggers else {} + timing = (trig.get("config") or {}).get("timing") or {} + ignored = [k for k in ("triggerTime", "weekdays", "daysOfMonth", "starting", "ending", "timezone", "tz") if k in timing] + out.append(result("scheduledTime 触发器且 timing 只含生效字段", trig.get("category") == "scheduledTime" and not ignored and "type" in timing, + "category=%s timing=%r ignored=%s" % (trig.get("category"), timing, ignored))) + warns = re.search(r"(忽略|不支持|仅|只).{0,30}(interval|间隔|type)|weekday|工作日.{0,10}(不|无法|忽略)|backend ignores", ans, re.I) + out.append(result("答复如实告知『具体时刻/星期』不被后端支持", bool(warns), ans[:250])) + claims_exact = re.search(r"(已.{0,6}(配好|设置).{0,40}(工作日|下午\s*5|17:00))|每个工作日下午 ?5 ?点.{0,10}(会|将)跑", ans) + out.append(result("未虚假承诺『工作日下午5点会照常执行』", not claims_exact, ans[:200])) + + elif ev_name == "secret-env-usage": + wf = find_wf(cfg, f"汇率查询{tag}") + if not wf: + out.append(result("创建名为 汇率查询%s 的工作流" % tag, False, "workflow list 中未找到")) + return out + wf_full, triggers, actions = wf_nodes(cfg, wf["id"]) + out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive"))) + secrets = cli_json(cfg, "secret", "list", wf["id"]) or [] + has_secret = bool(secrets) + out.append(result("存了密钥(非明文入代码;secret set 或 secret list 可见)", + any(re.search(r"secret set", c) for c in cmds) or has_secret, + [c for c in cmds if "secret set" in c][:1] or f"secret list: {len(secrets)} 条")) + script = next((a for a in actions if a.get("category") == "script"), None) + code = (script or {}).get("config", {}).get("code", "") + out.append(result("脚本经 process.env 读取密钥", bool(re.search(r"process\.env\.[A-Z_]+", code)), code[:150])) + out.append(result("脚本/工作流配置中无明文 key(sk-test-abc123)", "sk-test-abc123" not in json.dumps(wf_full, ensure_ascii=False), + "明文检查通过" if "sk-test-abc123" not in json.dumps(wf_full, ensure_ascii=False) else "配置含明文!")) + + elif ev_name == "script-npm-deps": + wf = find_wf(cfg, f"日期演示{tag}") + if not wf: + out.append(result("创建名为 日期演示%s 的工作流" % tag, False, "workflow list 中未找到")) + return out + wf_full, triggers, actions = wf_nodes(cfg, wf["id"]) + out.append(result("工作流已激活", wf_full.get("isActive") is True, "isActive=%s" % wf_full.get("isActive"))) + script = next((a for a in actions if a.get("category") == "script"), None) + code = (script or {}).get("config", {}).get("code", "") if script else "" + out.append(result("脚本使用 dayjs", "dayjs" in code, code[:150])) + out.append(result("依赖经 --dependencies 声明(配置含 dayjs)", "dayjs" in json.dumps((script or {}).get("config", {})), + json.dumps((script or {}).get("config", {}), ensure_ascii=False)[:200])) + out.append(result("知晓后端 npm PATH 前提(答复或命令体现)", any(re.search(r"--dependencies", c) for c in cmds) or bool(re.search(r"PATH|npm|后端|service", ans)), + ans[:180])) + + elif ev_name == "lowperm-403-handling": + ans_l = ans.lower() + perm_ack = bool(re.search(r"权限|403|无权|forbidden|没.{0,4}(资格|权限)|令牌|token.{0,12}(无|不|失效)|授权", ans, re.I)) + fake_success = bool(re.search(r"(已(经)?(建|创建|完成)[了成]?.{0,10}(表|快递)|(表|快递).{0,8}(已|成功)(建|创建)|成功创建.{0,6}表)", ans)) + cli_tried = any("inkreach-teable" in c for c in cmds) + out.append(result("尝试用 CLI 执行任务", cli_tried, [c for c in cmds if "inkreach-teable" in c][:1])) + out.append(result("识别出权限不足并如实说明", perm_ack and not fake_success, ans[:220])) + # 未陷入 403 循环(超过 8 次失败重试视为 flailing) + f403 = sum(1 for c in cmds if re.search(r"inkreach-teable", c)) + out.append(result("未陷入无意义重试(命令次数有界)", f403 <= 12, f"cli calls={f403}")) + + elif ev_name == "deep-pagination": + tbl13 = cfg.get("TBL13_ID") + if not tbl13: + out.append(result("fixture big_logs 存在", False, "TBL13_ID 未配置")) + return out + recs = (cli_json(cfg, "record", "get", "--table-id", tbl13, "--take", "1000") or {}).get("records") or [] + slow = [r for r in recs if str((r.get("fields") or {}).get("订单号", "")).startswith("SLOW-")] + ids = [str((r.get("fields") or {}).get("订单号")) for r in slow] + count_ok = bool(re.search(r"2\s*(条|个|records?|行)", ans)) + found_any = any(i in ans for i in ids) + out.append(result("正确给出 SLOW- 记录总数=2", count_ok, f"实际: {ids}")) + out.append(result("答复包含一条真实 SLOW- 记录的订单号", found_any, ans[:200])) + + elif ev_name == "vague-daily-notify": + # 模糊需求:每天 + 订单进展 + 站内通知 → scheduledTime + 统计 + sendNotification + active + wfs = cli_json(cfg, "workflow", "list") or [] + cands = [w for w in wfs if w.get("isActive")] + hit = None + for w in cands: + wf_full, trs, acs = wf_nodes(cfg, w["id"]) + if (trs and trs[0].get("category") == "scheduledTime" + and any(a.get("category") == "sendNotification" for a in acs)): + hit = wf_full + break + out.append(result("建了激活的每日定时工作流(scheduledTime+sendNotification)", hit is not None, + f"active scheduledTime+notify wf: {hit.get('name') if hit else None}")) + if hit: + _, trs, acs = wf_nodes(cfg, hit["id"]) + trig = trs[0] if trs else {} + timing = (trig.get("config") or {}).get("timing") or {} + out.append(result("timing 为生效字段(type+interval)", "type" in timing and "interval" in timing, f"timing={timing}")) + has_stats = any(a.get("category") == "script" and re.search(r"record|count|统计|length", (a.get("config") or {}).get("code", "")) for a in acs) + out.append(result("有订单统计逻辑", has_stats, [a.get("category") for a in acs])) + + elif ev_name == "field-options-update": + fields = cli_json(cfg, "field", "list", tbl) or [] + f = next((x for x in fields if x.get("name") == "结果"), {}) + choices = [c.get("name") if isinstance(c, dict) else c for c in ((f.get("options") or {}).get("choices") or [])] + out.append(result("「结果」字段 choices 新增「已取消」", "已取消" in choices, "choices=%s" % choices)) + out.append(result("使用 field update --options(走 convert API)", any(re.search(r"field update.*--options|field.*--options", c, re.S) for c in cmds), + [c for c in cmds if "field" in c and "update" in c][:1])) + out.append(result("原有选项未被破坏(成功/失败/处理中 仍在)", all(x in choices for x in ("成功", "失败", "处理中")), "choices=%s" % choices)) + + return out + + +def main(): + iteration = sys.argv[1] + tag = sys.argv[2] + iteration_dir = ROOT / iteration + for eval_dir in sorted(iteration_dir.glob("eval-*")): + ev_name = eval_dir.name.split("-", 2)[2] + for config_dir in sorted(p for p in eval_dir.iterdir() if p.is_dir()): + run_dir = config_dir / "run-1" + if not (run_dir / "pi.jsonl").exists(): + continue + cfg = env_all.get(config_dir.name) or env_all['with_skill'] + cfg_short = "W" if config_dir.name == "with_skill" else "O" + tag_cfg = f"{tag}{cfg_short}" + try: + expectations = grade_eval(cfg, ev_name, tag_cfg, run_dir, cfg_short) + except Exception as e: + expectations = [result("判分异常", False, repr(e))] + g = {"expectations": expectations} + (run_dir / "grading.json").write_text(json.dumps(g, indent=2, ensure_ascii=False)) + npass = sum(1 for x in expectations if x["passed"]) + print(f"{eval_dir.name:38} {config_dir.name:12} {npass}/{len(expectations)}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/evals/grade_multiturn.py b/evals/grade_multiturn.py new file mode 100644 index 0000000..b77966e --- /dev/null +++ b/evals/grade_multiturn.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Grade multi-turn (correction) evals: final state must satisfy the CORRECTED +requirement; amendment of the existing artifact is preferred over rebuilding. + +mt-1 accepted shapes for "结果 变成 失败 才站内通知": + A) recordCondition trigger with `conditions` (state-match reading), OR + B) recordUpdated trigger watching the 结果 field + script gate on 失败 + (transition reading — semantically at least as correct) +mt-2: incremental schema (add 渠道 select + mark the existing record). +""" +import json, os, re, subprocess, sys +from pathlib import Path + +HARNESS = Path(__file__).parent +ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun")) +env_all = json.loads(Path(os.environ.get("EVAL_ENV", str(HARNESS.parent / "sandbox-env.json"))).read_text()) +CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable") + + +def cli(*a): + env = {**os.environ, + "INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"], + "INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"], + "INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"]} + p = subprocess.run([CLI, *a, "--json"], capture_output=True, text=True, env=env, timeout=60) + try: + return json.loads(p.stdout) + except json.JSONDecodeError: + return None + + +def result(t, p, e): + return {"text": t, "passed": bool(p), "evidence": str(e)[:300]} + + +def cmds_of(rd, turn=2): + out = [] + f = rd / f"turn{turn}.jsonl" + if not f.exists(): + return out + for line in f.read_text(errors="replace").splitlines(): + try: + e = json.loads(line) + except json.JSONDecodeError: + continue + if e.get("type") == "tool_execution_start" and e.get("toolName") == "bash": + out.append(e.get("args", {}).get("command", "")) + return out + + +def grade_mt1(): + out = [] + wfs = cli("workflow", "list") or [] + if not isinstance(wfs, list): + return [result("workflow list 可读", False, wfs)] + ok = amended = False + for w in wfs: + full = cli("workflow", "get", w["id"]) or {} + trs = [n for n in full.get("nodes", []) if n.get("type") == "trigger"] + acs = [n for n in full.get("nodes", []) if n.get("type") == "action"] + trig = trs[0] if trs else {} + cfg = trig.get("config") or {} + cats = [a.get("category") for a in acs] + shape_a = trig.get("category") == "recordCondition" and "conditions" in cfg + shape_b = (trig.get("category") == "recordUpdated" and cfg.get("fieldIds") + and any(a.get("category") == "script" and re.search(r"失败", (a.get("config") or {}).get("code", "")) for a in acs)) + if full.get("isActive") and (shape_a or shape_b): + ok = "sendNotification" in cats and "sendEmail" not in cats + out.append(result("修正后终态:仅在结果变失败时站内通知(recordCondition 或 recordUpdated+门控)", ok, + [w.get("name") for w in wfs])) + c2 = cmds_of(ROOT / "iteration-mt" / "mt-1-notify-correction" / "auto_skill" / "run-1") + amended = any(re.search(r"trigger set|action (update|add|delete)", c) for c in c2) and not any( + re.search(r"workflow create", c) for c in c2) + out.append(result("第二轮是修改现有工作流而非推倒重建", amended, + [c[:80] for c in c2 if "workflow" in c][:3])) + return out + + +def grade_mt2(): + out = [] + tables = cli("table", "list") or [] + fb = [t for t in tables if "反馈" in t.get("name", "") or "feedback" in t.get("name", "").lower()] + if not fb: + return [result("反馈表存在", False, [t.get("name") for t in tables])] + tid = fb[0]["id"] + flds = cli("field", "list", tid) or [] + okf = any(f.get("name") == "内容" for f in flds) + ch = next((f for f in flds if f.get("name") == "渠道"), None) + okc = bool(ch) and set(["微信", "电话", "邮件"]) <= set( + c.get("name") for c in (ch.get("options") or {}).get("choices", [])) + recs = (cli("record", "get", "--table-id", tid) or {}).get("records") or [] + okr = any((r.get("fields") or {}).get("渠道") == "微信" for r in recs) + out.append(result("第一轮产物保留(表+内容字段)", okf, fb[0]["name"])) + out.append(result("第二轮增量:渠道字段三选项", okc, (ch or {}).get("options"))) + out.append(result("示例记录渠道=微信", okr, [(r.get("fields") or {}).get("订单号"), (r.get("fields") or {}).get("渠道")] if not okr else "ok")) + return out + + +def main(): + iteration = sys.argv[1] if len(sys.argv) > 1 else "iteration-mt" + g1 = grade_mt1() + g2 = grade_mt2() + for name, exps in [("mt-1-notify-correction", g1), ("mt-2-incremental-schema", g2)]: + rd = ROOT / iteration / name / "auto_skill" / "run-1" + (rd / "grading.json").write_text(json.dumps({"expectations": exps}, indent=2, ensure_ascii=False)) + np = sum(1 for x in exps if x["passed"]) + print(f"{name:28} {np}/{len(exps)}") + + +if __name__ == "__main__": + main() diff --git a/evals/grade_plugin_evals.py b/evals/grade_plugin_evals.py new file mode 100644 index 0000000..be1ef5f --- /dev/null +++ b/evals/grade_plugin_evals.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Grade plugin-skill eval runs (ported from plugin-workspace, per-config sandboxes).""" +import json, os, re, subprocess, sys +from pathlib import Path + +HARNESS = Path(__file__).parent # evals/ itself +ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun")) +CLI = os.environ.get("EVAL_CLI", "/root/eval-rerun/bin-new/bin/inkreach-teable") +env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'plugin-sandbox-env.json'))).read_text()) + +FORBIDDEN_IMPORT = re.compile(r"^\s*import\s+(?!type)[^\n]*from\s+['\"]@teable/(sdk|ui-lib|openapi|core|icons)", re.M) +REACT_IMPORT = re.compile(r"^\s*import\s+(?!type)[^\n]*from\s+['\"]react['\"]", re.M) + + +def cli_json(*args): + e = {**os.environ, + "INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"], + "INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"], + "INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"]} + p = subprocess.run([CLI, *args, "--json"], capture_output=True, text=True, env=e, timeout=60) + try: + return json.loads(p.stdout) + except json.JSONDecodeError: + return None + + +def transcript(run_dir: Path): + cmds = [] + for line in (run_dir / "pi.jsonl").read_text(errors="replace").splitlines(): + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + if ev.get("type") == "tool_execution_start" and ev.get("toolName") == "bash": + cmds.append(ev.get("args", {}).get("command", "")) + return cmds + + +def answer(run_dir: Path) -> str: + fa, af = run_dir / "final_answer.md", run_dir / "assistant_final.md" + return fa.read_text(errors="replace") if fa.exists() else af.read_text(errors="replace") + + +def plugin_sources(run_dir: Path): + files = sorted((run_dir / "plugin").rglob("*.[tj]s*")) or sorted(run_dir.rglob("plugin.json")) + return "\n".join(p.read_text(errors="replace") for p in files), files + + +def result(text, passed, evidence): + return {"text": text, "passed": bool(passed), "evidence": str(evidence)[:400]} + + +def grade_eval(ev_name, run_dir: Path, cfg_short: str): + cmds = transcript(run_dir) + ans = answer(run_dir) + out = [] + + if ev_name in ("contextmenu-plugin", "panel-widget"): + src, files = plugin_sources(run_dir) + want_pos = "contextMenu" if ev_name == "contextmenu-plugin" else "panel" + want_api = "addContextMenuAction" if ev_name == "contextmenu-plugin" else "addPanel" + pj = run_dir / "plugin" / "plugin.json" + manifest_ok = False + if pj.exists(): + try: + m = json.loads(pj.read_text()) + manifest_ok = bool(re.fullmatch(r"[a-z0-9-]+", m.get("id", ""))) and want_pos in (m.get("positions") or []) + except json.JSONDecodeError: + pass + out.append(result(f"plugin.json 存在且 manifest 合法(id 合规、positions 含 {want_pos})", manifest_ok, + pj.read_text(errors="replace")[:150] if pj.exists() else "plugin/plugin.json 未找到")) + out.append(result("代码从 window.__TEABLE_HOST__ 获取宿主能力", "__TEABLE_HOST__" in src, + "host global: %s" % ("有" if "__TEABLE_HOST__" in src else "无"))) + bad = FORBIDDEN_IMPORT.findall(src) + out.append(result("@teable/* 无运行时值导入(仅允许 import type)", not bad, "违规: %r" % bad if bad else "ok")) + uses_react = bool(REACT_IMPORT.search(src)) + wp = run_dir / "plugin" / "webpack.config.js" + wp_txt = wp.read_text(errors="replace") if wp.exists() else "" + externals_ok = "__TEABLE_HOST__" in wp_txt and "externals" in wp_txt + out.append(result("react 导入经 externals 映射(或无 react 裸导入)", externals_ok if uses_react else True, + "webpack externals: %s" % ("映射" if externals_ok else ("未映射!" if uses_react else "不需要")))) + out.append(result(f"definePlugin 注册且调用 ctx.{want_api}", "definePlugin" in src and want_api in src, + "definePlugin=%s %s=%s" % ("definePlugin" in src, want_api, want_api in src))) + if ev_name == "contextmenu-plugin": + edits = re.search(r"updateRecords|record.*update|OpenAPI", src, re.I) + out.append(result("实现了批量更新记录逻辑", bool(edits), edits.group(0) if edits else "未发现")) + else: + sel = re.search(r"useSelection|selection|getSelectionRecords", src, re.I) + out.append(result("通过宿主 SDK 获取选中记录", bool(sel), sel.group(0) if sel else "未发现")) + + elif ev_name == "deploy-chain": + plugin_id = f"eval-deploy-{cfg_short}" + up = any("upload-bundle" in c for c in cmds) + out.append(result("调用了 upload-bundle 上传", up, [c for c in cmds if "upload-bundle" in c][:1])) + enabled = any("plugin-config" in c for c in cmds) + out.append(result("启用了 plugin-config", enabled, [c for c in cmds if "plugin-config" in c][:1])) + installed = any(re.search(r"context-menu.*install|plugin-context-menu", c) for c in cmds) + out.append(result("安装到表级 context-menu 位置", installed, [c for c in cmds if "context-menu" in c][:1])) + cleaned = any(("DELETE" in c.upper() and "/api/plugin/" in c) or re.search(r"plugin uninstall|plugin-install", c) for c in cmds) \ + or (cli_json("plugin", "list") or [] and not any(p.get("id") == plugin_id for p in (cli_json("plugin", "list") or [])) and any("install" in c for c in cmds)) + out.append(result("执行了清理(卸载安装/删除插件)", cleaned, [c for c in cmds if "uninstall" in c or "DELETE" in c.upper()][:2])) + lst = json.dumps(cli_json("plugin", "list") or [], ensure_ascii=False) + out.append(result("实例上无插件残留(plugin list 不含该 id)", plugin_id not in lst, + "残留!" if plugin_id in lst else "干净")) + verify = re.search(r"已确认|确认(安装|清理|删除)|installed|verified|清理(完成|干净)", ans) + out.append(result("答复确认了安装与清理结果", bool(verify), ans[:200])) + + elif ev_name == "proxy-rule": + out.append(result("方案使用同源代理 /api/plugin-proxy//...", "/api/plugin-proxy/" in ans, ans[:250])) + out.append(result("指出需注册 PLUGIN_API_SERVICES 并重启 teable-backend", "PLUGIN_API_SERVICES" in ans, + "提及" if "PLUGIN_API_SERVICES" in ans else "未提及")) + lan_fetch = re.search(r"fetch\(\s*['\"]http://192\.168", ans) + out.append(result("不建议在 bundle 里直接请求局域网 IP", lan_fetch is None, lan_fetch.group(0) if lan_fetch else "ok")) + + elif ev_name == "pitfall-diagnosis": + cause = re.search(r"(@teable/sdk|值导入|运行时导入|externals|undefined)|import", ans) and re.search(r"__TEABLE_HOST__|宿主全局|host global", ans) + out.append(result("定位为 @teable/* 运行时导入被 externals 映射为 undefined", bool(cause), ans[:250])) + out.append(result("给出修复:改从 window.__TEABLE_HOST__ 读取(或仅 import type)", "__TEABLE_HOST__" in ans, ans[:200])) + + elif ev_name == "native-form-routing": + nf = re.search(r"NativeForm|useNativeForm|NativeFormScope", ans) + out.append(result("指出原生表单注入能力(NativeForm/NativeFormScope/useNativeForm)", bool(nf), + nf.group(0) if nf else ans[:200])) + # 咨询类任务应给出可实现方案(两档之一 + 关键代码/props) + plan = bool(nf) and bool(re.search(r"visibleFieldIds|slots|NativeFormField|formId|interceptSubmit", ans)) + out.append(result("给出了具体实现要点(props/多步结构)", plan, ans[:250])) + + return out + + +def main(): + iteration_dir = ROOT / sys.argv[1] + for eval_dir in sorted(iteration_dir.glob("eval-*")): + ev_name = eval_dir.name.split("-", 2)[2] + for config_dir in sorted(p for p in eval_dir.iterdir() if p.is_dir()): + run_dir = config_dir / "run-1" + if not (run_dir / "pi.jsonl").exists(): + continue + cfg_short = "W" if config_dir.name == "with_skill" else "O" + try: + expectations = grade_eval(ev_name, run_dir, cfg_short) + except Exception as e: + expectations = [result("判分异常", False, repr(e))] + (run_dir / "grading.json").write_text(json.dumps({"expectations": expectations}, indent=2, ensure_ascii=False)) + npass = sum(1 for x in expectations if x["passed"]) + print(f"{eval_dir.name:34} {config_dir.name:12} {npass}/{len(expectations)}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/evals/plugin-evals.json b/evals/plugin-evals.json new file mode 100644 index 0000000..db05579 --- /dev/null +++ b/evals/plugin-evals.json @@ -0,0 +1,41 @@ +{ + "skill_name": "inkreach-teable-plugin", + "evals": [ + { + "id": 1, + "name": "contextmenu-plugin", + "prompt": "帮我写一个右键菜单插件「批量置成功」:在表格里选中若干记录后右键,一键把这些记录的「结果」字段全部改成「成功」。把完整可构建的项目文件写到当前目录 plugin/ 子目录(plugin.json、webpack.config.js、src/…),手写即可,不必运行构建,也不用部署。", + "expected_output": "manifest 含 id/positions contextMenu;代码从 window.__TEABLE_HOST__ 取 SDK/OpenAPI/React;@teable/* 仅 import type 或不出现(禁止运行时值导入);definePlugin + ctx.addContextMenuAction;webpack externals 把 react 映射到 host 全局。" + }, + { + "id": 2, + "name": "panel-widget", + "prompt": "写一个侧边面板插件「选中统计」:打开面板时实时显示当前表名和当前选中记录数量。项目文件写到 plugin/ 子目录,手写即可,不必构建部署。", + "expected_output": "positions panel;ctx.addPanel;选中数与表名来自 host SDK(useSelection/useTableId 等,经 window.__TEABLE_HOST__);无禁止的运行时导入。" + }, + { + "id": 3, + "name": "deploy-chain", + "prompt": "当前目录 ready-plugin/ 下是我准备好的插件(plugin.json + dist/main.js)。把它部署到实例并安装到 {TBL} 这张表的右键菜单,确认安装成功;然后完整清理:卸载安装、停用、删除插件,确认都清干净。", + "expected_output": "upload-bundle(201)→ plugin-config 启用 → context-menu install → 验证;然后卸载安装 + DELETE /api/plugin/:id;plugin list 确认无残留。" + }, + { + "id": 4, + "name": "proxy-rule", + "prompt": "我的插件需要调用我局域网的后端服务 http://192.168.124.92:8001/api/track 拿数据,实例是公网访问的(table.inkreach.cc),公网用户也要能用。插件里的请求应该怎么写?需要在实例侧改什么?给出说明和关键代码。", + "expected_output": "fetch('/api/plugin-proxy//api/track') 同源代理;注册 PLUGIN_API_SERVICES(两个 env 文件)+ 重启 teable-backend;不在 bundle 里硬编码局域网 IP;外部源需 manifest apiOrigins。" + }, + { + "id": 5, + "name": "pitfall-diagnosis", + "prompt": "我上传的插件右键菜单能出现,但点击就白屏,控制台报:Cannot read properties of undefined (reading 'useFields')。帮我定位原因并给出修复。", + "expected_output": "定位为运行时从 @teable/sdk 等 import(被 webpack externals 映射成 undefined)/未从 window.__TEABLE_HOST__ 读取;修复=改用 host 全局取 SDK 或仅 import type。" + }, + { + "id": 6, + "name": "native-form-routing", + "prompt": "我想在插件里做一个自定义布局的多步表单页,每一步嵌入系统原生的字段编辑器(不用自己重写输入组件),能做到吗?怎么实现?", + "expected_output": "指出用原生表单注入能力:NativeForm / NativeFormScope / useNativeForm(sdk-api.md 有专门章节);说明分层(Tier-1 注入/Tier-2 kit)。" + } + ] +} diff --git a/evals/run_evals4.py b/evals/run_evals4.py new file mode 100644 index 0000000..f56ba19 --- /dev/null +++ b/evals/run_evals4.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Launch inkreach-teable skill evals (iteration-4 A/B rerun). + +Usage: python3 run_evals4.py [eval-ids] + config-spec: with_skill=/root/eval-rerun/skill-new:/root/eval-rerun/bin-new/bin,old_skill=/root/eval-rerun/skill-old:/root/eval-rerun/bin-old/bin + +Differences vs the original run_evals.py (eval-integrity fixes, see LESSONS.md): +- evals.json + sandbox-env.json live in THIS harness dir (outside the agent's cwd tree) +- per-config sandbox BASE via INKREACH_TEABLE_BASE_ID (no cross-config contamination) +- per-config PATH prepended with that config's CLI version dir (0.1.4 vs 0.1.5) +- pi runs with an isolated HOME (no user-level skills discovered) +- grading.json written only afterwards by grade_evals4.py (no answers near agents) +""" +import json +import os +import re +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +HARNESS = Path(__file__).parent # evals/ itself +ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun")) +MODEL = os.environ.get("EVAL_MODEL", "opencode-go/deepseek-v4.1-flash") +RUN_TIMEOUT = 600 +PI_HOME = os.environ.get("EVAL_PI_HOME", "/root/eval-rerun/pi-home") + +env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'sandbox-env.json'))).read_text()) + +PROMPT_TMPL = """你在操作一个已部署的 inkreach-teable 实例(Teable fork,自研自动化引擎)。 + +环境(已配置好,直接使用): +- CLI 命令: `inkreach-teable`(已在 PATH)。endpoint / token / base 已通过环境变量 INKREACH_TEABLE_ENDPOINT / INKREACH_TEABLE_TOKEN / INKREACH_TEABLE_BASE_ID 注入,不要执行 auth login,不要改配置文件。 +- 只操作环境变量指定的这个基地(评估沙箱);严禁触碰其他基地。 +- 工作目录就是当前目录,把中间产物写在这里。 + +{SKILL_LINE} + +任务:{TASK} + +要求: +1. 逐步执行任务,用 CLI 完成实际操作(不是只给命令)。 +2. 完成后把最终答复写入 final_answer.md:做了什么、关键命令、结果、需要用户知道的事项。""" + + +def extract_final_text(jsonl_path: Path) -> str: + texts = [] + for line in jsonl_path.read_text(errors="replace").splitlines(): + line = line.strip() + if not line: + continue + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + if ev.get("type") == "turn_end" and ev.get("message", {}).get("role") == "assistant": + for c in ev["message"].get("content", []): + if c.get("type") == "text" and c.get("text", "").strip(): + texts.append(c["text"]) + return texts[-1] if texts else "" + + +def extract_usage(jsonl_path: Path) -> dict: + usage = {} + for line in jsonl_path.read_text(errors="replace").splitlines(): + m = re.search(r'"usage":\{[^{}]*"totalTokens":(\d+)', line) + if m: + usage["totalTokens"] = int(m.group(1)) + return usage + + +def run_one(task, config, skill_path, bin_dir, cfg_env, run_dir: Path): + run_dir.mkdir(parents=True, exist_ok=True) + if config == "auto_skill": + # trigger test: no skill hint at all; the skill must be DISCOVERED from + # the user-level skills dir of the isolated (skilled) HOME. + skill_line = "" + elif skill_path: + skill_line = ( + f"- 先完整阅读技能说明 {skill_path}/SKILL.md 并严格按其路由规则与关键约束操作;\n" + f" 需要细节时再按其中链接查阅 guides/ 与 api-reference/ 文档。" + ) + else: + skill_line = "- 环境里没有额外文档,自行判断。" + prompt = PROMPT_TMPL.format(SKILL_LINE=skill_line, TASK=task) + (run_dir / "prompt.txt").write_text(prompt) + + full_env = { + **os.environ, + "HOME": (os.environ.get("EVAL_PI_HOME_SKILLED", "/root/eval-rerun/pi-home-skilled") if config == "auto_skill" else PI_HOME), + "PATH": f"{bin_dir}:{os.environ['PATH']}", + **{k: env_all[k] for k in ("INKREACH_TEABLE_ENDPOINT", "INKREACH_TEABLE_TOKEN")}, + **cfg_env, + } + cmd = ["timeout", str(RUN_TIMEOUT), "pi", "-p", "--mode", "json", "--model", MODEL, prompt] + t0 = time.time() + with open(run_dir / "pi.jsonl", "w") as out: + proc = subprocess.run(cmd, stdout=out, stderr=subprocess.DEVNULL, cwd=run_dir, env=full_env) + dur = time.time() - t0 + timing = {"duration_ms": int(dur * 1000), "total_duration_seconds": round(dur, 1), + "total_tokens": extract_usage(run_dir / "pi.jsonl").get("totalTokens"), + "exit_code": proc.returncode, "cli": Path(bin_dir).name, "config": config} + (run_dir / "timing.json").write_text(json.dumps(timing, indent=2)) + (run_dir / "assistant_final.md").write_text(extract_final_text(run_dir / "pi.jsonl") or "(no final text captured)") + return config, run_dir.name, proc.returncode, round(dur, 1) + + +def main(): + iteration = sys.argv[1] + tag = sys.argv[2] + iteration_dir = ROOT / iteration + configs = {} + for part in sys.argv[3].split(","): + name, _, rest = part.partition("=") + skill_path, _, bin_dir = rest.partition(":") + configs[name] = (skill_path, bin_dir) + + evals = json.loads(Path(os.environ.get('EVAL_EVALS', str(HARNESS / 'teable-evals.json'))).read_text())["evals"] + if len(sys.argv) > 4 and sys.argv[4]: + only = {int(x) for x in sys.argv[4].split(",")} + evals = [e for e in evals if e["id"] in only] + + jobs = [] + for ev in evals: + name = f"eval-{ev['id']}-{ev['name']}" + for config, (skill_path, bin_dir) in configs.items(): + cfg = env_all.get(config) or env_all['with_skill'] + cfg_short = "W" if config == "with_skill" else "O" + task = (ev["prompt"] + .replace("{TAG}", f"{tag}{cfg_short}") + .replace("{WFID}", cfg["WF_ID"]) + .replace("{TBL5}", cfg["TBL5_NAME"]) + .replace("{ORD}", cfg["ORD"])) + meta = {"eval_id": ev["id"], "eval_name": ev["name"], "config": config, "prompt": task, "assertions": []} + rd = iteration_dir / name / config / "run-1" + rd.parent.mkdir(parents=True, exist_ok=True) + (rd.parent / "eval_metadata.json").write_text(json.dumps(meta, indent=2, ensure_ascii=False)) + cfg_env = { + "INKREACH_TEABLE_BASE_ID": cfg["INKREACH_TEABLE_BASE_ID"], + "INKREACH_TEABLE_EVAL_TBL": cfg["TBL5_ID"], + "INKREACH_TEABLE_EVAL_ORD": cfg["ORD"], + } + if ev["id"] == 12: + cfg_env["INKREACH_TEABLE_TOKEN"] = env_all["LOWPERM_TOKEN"] + jobs.append((task, config, skill_path, bin_dir, cfg_env, rd)) + + print(f"{len(jobs)} runs -> {iteration_dir}", flush=True) + with ThreadPoolExecutor(max_workers=6) as ex: + futs = [ex.submit(run_one, *j) for j in jobs] + for f in futs: + print(f.result(), flush=True) + + +if __name__ == "__main__": + main() diff --git a/evals/run_multiturn.py b/evals/run_multiturn.py new file mode 100644 index 0000000..6877c1f --- /dev/null +++ b/evals/run_multiturn.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Multi-turn evals: two-round conversations where turn 2 corrects turn 1. + +Verifies the agent AMENDS the previous artifacts (edit workflow / add field) +instead of ignoring the correction or rebuilding blindly. + +Usage: python3 run_multiturn.py +""" +import json, os, subprocess, sys, time +from pathlib import Path + +HARNESS = Path(__file__).parent +ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun")) +MODEL = os.environ.get("EVAL_MODEL", "opencode-go/deepseek-v4.1-flash") +RUN_TIMEOUT = 480 +PI_HOME = os.environ.get("EVAL_PI_HOME_SKILLED", "/root/eval-rerun/pi-home-skilled") # trigger mode: skill auto-discovered + +env_all = json.loads((HARNESS / "sandbox-env.json").read_text()) + +TASKS = [ + { + "name": "mt-1-notify-correction", + "turn1": "帮我建个自动化:order_logs 表有新记录的时候通知我。名字你来定,配好激活。", + "turn2": "不对,改一下:别用邮件,要站内通知;而且只在「结果」字段变成「失败」的时候才通知,不是有新记录就通知。", + "turn1_hint": None, + }, + { + "name": "mt-2-incremental-schema", + "turn1": "帮我建张表存客户反馈,加个单行文本「内容」字段,随便插一条示例。", + "turn2": "再补一个单选字段「渠道」,选项:微信/电话/邮件。然后把刚才那条示例记录的渠道标成「微信」。", + "turn1_hint": None, + }, +] + +WRAP = """要求: +1. 逐步用 CLI 完成实际操作(不是只给命令)。 +2. 每轮结束后把当前轮的答复写入 final_answer_turn{N}.md(做了什么、关键命令、结果)。 +3. 用户的后续修正优先于之前的做法;能改就改现有的,不要无视.""" + + +def turn(run_dir: Path, sid: str, text: str, n: int): + prompt = f"{text}\n\n{WRAP.format(N=n)}" + cmd = ["timeout", str(RUN_TIMEOUT), "pi", "-p", "--mode", "json", + "--session-id", sid, "--model", MODEL, prompt] + out_f = run_dir / f"turn{n}.jsonl" + env = { + **os.environ, "HOME": PI_HOME, + "PATH": f"/root/eval-rerun/bin-new/bin:{os.environ['PATH']}", + "INKREACH_TEABLE_ENDPOINT": env_all["INKREACH_TEABLE_ENDPOINT"], + "INKREACH_TEABLE_TOKEN": env_all["INKREACH_TEABLE_TOKEN"], + "INKREACH_TEABLE_BASE_ID": env_all["with_skill"]["INKREACH_TEABLE_BASE_ID"], + } + with open(out_f, "w") as out: + p = subprocess.run(cmd, stdout=out, stderr=subprocess.DEVNULL, cwd=run_dir, env=env) + # pull last assistant text + last = "" + for line in out_f.read_text(errors="replace").splitlines(): + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + if ev.get("type") == "turn_end" and ev.get("message", {}).get("role") == "assistant": + for c in ev["message"].get("content", []): + if c.get("type") == "text" and c.get("text", "").strip(): + last = c["text"] + (run_dir / f"final_answer_turn{n}.md").write_text(last or "(none)") + return p.returncode + + +def main(): + iteration, tag = sys.argv[1], sys.argv[2] + for t in TASKS: + rd = ROOT / iteration / t["name"] / "auto_skill" / "run-1" + rd.mkdir(parents=True, exist_ok=True) + sid = f"mt-{tag}-{t['name']}" + t0 = time.time() + r1 = turn(rd, sid, t["turn1"], 1) + r2 = turn(rd, sid, t["turn2"], 2) + print(f"{t['name']}: turn1 exit={r1} turn2 exit={r2} dur={round(time.time()-t0,1)}s", flush=True) + + +if __name__ == "__main__": + main() diff --git a/evals/run_plugin_evals.py b/evals/run_plugin_evals.py new file mode 100644 index 0000000..616e142 --- /dev/null +++ b/evals/run_plugin_evals.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Launch inkreach-teable-plugin skill evals (integrity-clean port). + +Usage: python3 run_plugin_evals.py [eval-ids] + config-spec: with_skill=/root/eval-rerun/skill-plugin-new:/root/eval-rerun/bin-new/bin +""" +import json, os, re, shutil, subprocess, sys, time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +HARNESS = Path(__file__).parent # evals/ itself +ROOT = Path(os.environ.get("EVAL_ROOT", "/root/eval-rerun")) +MODEL = os.environ.get("EVAL_MODEL", "opencode-go/deepseek-v4.1-flash") +RUN_TIMEOUT = 900 # plugin evals build bundles — need more headroom +PI_HOME = os.environ.get("EVAL_PI_HOME", "/root/eval-rerun/pi-home") +FIXTURE = "/root/pi-skill-lab/plugin-workspace/fixture-deploy-W" + +env_all = json.loads(Path(os.environ.get('EVAL_ENV', str(HARNESS / 'plugin-sandbox-env.json'))).read_text()) + +PROMPT_TMPL = """你在为一套已部署的 inkreach-teable 实例(Teable fork)开发/部署动态 React 插件。 + +环境(已配置好,直接使用): +- 后端 http://localhost:3003,前端 http://localhost:3002。PAT 已通过环境变量 INKREACH_TEABLE_ENDPOINT / INKREACH_TEABLE_TOKEN 注入(`inkreach-teable` CLI 可直接用)。 +- 评估沙箱基地 ID:{BASE}。凡涉及实例的读写一律只在这个基地内,严禁触碰其他基地;严禁向实例的插件注册表上传与任务无关的演示插件。 +- 工作目录就是当前目录,所有产出写在这里。 + +{SKILL_LINE} + +任务:{TASK} + +要求: +1. 按任务性质完成:咨询类问题(能不能/怎么做)直接给出方案与关键代码即可,不必搭建完整工程;开发/部署类任务才需要实际构建与部署。 +2. 完成后把最终答复写入 final_answer.md:结论、关键实现/命令、结果、需要用户知道的事项。""" + + +def extract_final_text(p: Path) -> str: + texts = [] + for line in p.read_text(errors="replace").splitlines(): + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + if ev.get("type") == "turn_end" and ev.get("message", {}).get("role") == "assistant": + for c in ev["message"].get("content", []): + if c.get("type") == "text" and c.get("text", "").strip(): + texts.append(c["text"]) + return texts[-1] if texts else "" + + +def run_one(task, config, skill_path, bin_dir, cfg_env, run_dir: Path): + run_dir.mkdir(parents=True, exist_ok=True) + skill_line = ( + f"- 先完整阅读技能说明 {skill_path}/SKILL.md 并按其模块地图(按需读 dynamic-plugins/ 下文档)操作。" + if skill_path else "" + ) + prompt = PROMPT_TMPL.format(BASE=cfg_env["INKREACH_TEABLE_BASE_ID"], SKILL_LINE=skill_line, TASK=task) + (run_dir / "prompt.txt").write_text(prompt) + full_env = { + **os.environ, "HOME": PI_HOME, "PATH": f"{bin_dir}:{os.environ['PATH']}", + **{k: env_all[k] for k in ("INKREACH_TEABLE_ENDPOINT", "INKREACH_TEABLE_TOKEN")}, + **cfg_env, + } + cmd = ["timeout", str(RUN_TIMEOUT), "pi", "-p", "--mode", "json", "--model", MODEL, prompt] + t0 = time.time() + with open(run_dir / "pi.jsonl", "w") as out: + proc = subprocess.run(cmd, stdout=out, stderr=subprocess.DEVNULL, cwd=run_dir, env=full_env) + (run_dir / "timing.json").write_text(json.dumps( + {"duration_s": round(time.time() - t0, 1), "exit_code": proc.returncode, "config": config})) + (run_dir / "assistant_final.md").write_text(extract_final_text(run_dir / "pi.jsonl") or "(none)") + return config, run_dir.name, proc.returncode + + +def main(): + iteration, tag = sys.argv[1], sys.argv[2] + iteration_dir = ROOT / iteration + configs = {} + for part in sys.argv[3].split(","): + name, _, rest = part.partition("=") + sp, _, bd = rest.partition(":") + configs[name] = (sp, bd) + evals = json.loads(Path(os.environ.get('EVAL_EVALS', str(HARNESS / 'plugin-evals.json'))).read_text())["evals"] + if len(sys.argv) > 4 and sys.argv[4]: + only = {int(x) for x in sys.argv[4].split(",")} + evals = [e for e in evals if e["id"] in only] + + jobs = [] + for ev in evals: + name = f"eval-{ev['id']}-{ev['name']}" + for config, (skill_path, bin_dir) in configs.items(): + cfg = env_all[config] + cfg_short = "W" if config == "with_skill" else "O" + task = ev["prompt"].replace("{TBL}", cfg["TBL"]) + rd = iteration_dir / name / config / "run-1" + rd.parent.mkdir(parents=True, exist_ok=True) + # eval-3 needs the ready-plugin fixture bundle in cwd + if ev["id"] == 3: + rp = rd / "ready-plugin" + if rp.exists(): + shutil.rmtree(rp) + shutil.copytree(FIXTURE, rp) + (rd.parent / "eval_metadata.json").write_text(json.dumps( + {"eval_id": ev["id"], "prompt": task, "assertions": []}, ensure_ascii=False)) + cfg_env = {"INKREACH_TEABLE_BASE_ID": cfg["INKREACH_TEABLE_BASE_ID"]} + jobs.append((task, config, skill_path, bin_dir, cfg_env, rd)) + + print(f"{len(jobs)} runs -> {iteration_dir}", flush=True) + with ThreadPoolExecutor(max_workers=3) as ex: # builds are heavy — 3 workers + for f in [ex.submit(run_one, *j) for j in jobs]: + print(f.result(), flush=True) + + +if __name__ == "__main__": + main() diff --git a/evals/teable-evals.json b/evals/teable-evals.json new file mode 100644 index 0000000..c71034c --- /dev/null +++ b/evals/teable-evals.json @@ -0,0 +1,86 @@ +{ + "skill_name": "inkreach-teable", + "evals": [ + { + "id": 1, + "name": "button-click-automation", + "prompt": "帮我在 order_logs 表里加一个按钮字段(名字用:催单按钮{TAG}),点这个按钮的时候触发一个自动化,自动化里跑一段脚本,脚本返回当前记录的订单号。然后激活它,创建一条新记录并点一下按钮测试,确认执行结果。", + "expected_output": "创建 button 字段(options.workflow 为对象 {id,...})、workflow、buttonClick 触发器、读取 triggerInput 的 script action、activate、fire button-click 测试、runs 确认 succeeded。" + }, + { + "id": 2, + "name": "scheduled-digest", + "prompt": "我想要每天定时跑一次自动化(工作流名字用:每日统计{TAG}),统计 order_logs 表里的记录数并输出。帮我配好。", + "expected_output": "scheduledTime 触发器只配 timing.type + timing.interval(后端忽略其他时间字段);script/getRecords action;activate。" + }, + { + "id": 3, + "name": "record-condition-email", + "prompt": "当 order_logs 表里有记录的「结果」字段变成「失败」时,自动发邮件通知 ops@xingchen.tech(工作流名字用:失败告警{TAG})。帮我配好。", + "expected_output": "recordCondition 触发器(分类是 recordCondition,条件键名是 conditions 而非 filter);sendEmail action 收件人 ops@xingchen.tech;activate。" + }, + { + "id": 4, + "name": "debug-failed-run", + "prompt": "我有个自动化 wfl5F8dQRDloo28w909……不对,ID 是 {WFID},名字叫 nightly-sync,最近好像没正常跑,帮我看看是什么情况,别急着改,先诊断清楚再说。", + "expected_output": "先 workflow get 查 isActive → runs list --status failed → runs get 看 nodeResults 错误;指出脚本抛错(boom: upstream sync timeout);未盲目重新激活。" + }, + { + "id": 5, + "name": "select-constraint-write", + "prompt": "在 {TBL5} 表里,把订单号为 {ORD} 的那条记录的「结果」改成「已完成」。另外告诉我这个「结果」字段目前一共有哪些可选值。", + "expected_output": "先查约束发现「已完成」不在 choices;要么说明并停下,要么先加选项再写入;不发生静默失败。" + }, + { + "id": 6, + "name": "official-cli-near-miss", + "prompt": "用 teable 的 CLI 工具帮我列出这个基地里都有哪些表。", + "expected_output": "路由到 inkreach-teable CLI(官方 teable CLI 在该实例自动化契约不兼容);列出 order_logs;未安装/调用官方 teable CLI 打实例。" + }, + { + "id": 7, + "name": "webhook-receiver", + "prompt": "帮我建一个自动化(名字:回单通知{TAG}),用 webhook 触发:外部系统会 POST 调用它。建好激活后,把可以调用的完整 URL 给我,并实际测试一次确认能通。", + "expected_output": "webhook 触发器;URL 形如 {endpoint}/api/webhook/{baseId}/{workflowId};用 fire webhook 或 curl 实测;已激活。" + }, + { + "id": 8, + "name": "weekday-schedule-trap", + "prompt": "我要一个自动化(名字:工作日盘点{TAG}),每个工作日下午5点跑一次,统计 order_logs 表的记录条数就行。帮我配好。", + "expected_output": "知道后端只认 timing.type+interval:要么明确告知『具体时刻/星期』不被支持并给出周期间隔替代方案;绝不悄悄配 weekdays/triggerTime 假装生效。timing 配置无被忽略字段。" + }, + { + "id": 9, + "name": "secret-env-usage", + "prompt": "建一个自动化(名字:汇率查询{TAG}),脚本里调用外部汇率 API,API key 是 sk-test-abc123,不要明文写在脚本里。配好激活即可,不用真跑通外部接口。", + "expected_output": "secret set 存 key(如 EXCHANGE_KEY)→ 脚本读 process.env.EXCHANGE_KEY;代码中不出现明文 key;激活。" + }, + { + "id": 10, + "name": "script-npm-deps", + "prompt": "我有个自动化脚本要用 dayjs 处理日期格式化。帮我建个演示工作流(名字:日期演示{TAG}),脚本里 require dayjs 并格式化当前日期输出,把依赖正确声明好。", + "expected_output": "action add/update 带 --dependencies '[{\"name\":\"dayjs\",\"version\":...}]'(或 dayjs@版本);知晓后端需 npm 在 service PATH(风险提示)。" + }, + { + "id": 11, + "name": "field-options-update", + "prompt": "把 order_logs 表「结果」字段加一个新选项「已取消」。就这一件事。", + "expected_output": "field update --options(走 convert API;普通 PATCH 会丢 options);验证 choices 里出现「已取消」。" + }, + { + "id": 12, + "name": "lowperm-403-handling", + "prompt": "帮我在当前基地里建一张新表,名字叫快递跟踪,加个单行文本字段 运单号,再加个数字字段 金额。建好后随便插一条示例数据。" + }, + { + "id": 13, + "name": "deep-pagination", + "prompt": "big_logs 这张表里记录挺多的。帮我把订单号以 SLOW- 开头的记录全部找出来:一共几条?再挑其中一条,把它的完整字段值原样发我。" + }, + { + "id": 14, + "name": "vague-daily-notify", + "prompt": "我想大概每天知道一下订单进展。邮件先不用,站内通知就行。你看着帮我弄一个。" + } + ] +} \ No newline at end of file diff --git a/package.json b/package.json index e19855d..72a27ed 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@inkreach/inkreach-teable", - "version": "0.1.8", + "version": "0.1.9", "description": "CLI (command: inkreach-teable) for operating inkreach-teable: tables, fields, records, views, and the self-built automation engine (8 triggers + 7 actions).", "license": "MIT", "type": "commonjs", diff --git a/scripts/smoke-commands.sh b/scripts/smoke-commands.sh index 7b83f6b..a40343b 100755 --- a/scripts/smoke-commands.sh +++ b/scripts/smoke-commands.sh @@ -51,8 +51,12 @@ cleanup() { curl -s -o /dev/null -X DELETE -H "Authorization: Bearer $TOK" "$EP/ trap cleanup EXIT echo "sandbox base: $BASE" -# ---------- auth / base ---------- +# ---------- auth / base / space ---------- smoke "auth status" $CLI auth status +smoke "space list" $CLI space list +SB2=$($CLI base create --space-id "$SPACE" --name cli-smoke-b2 --json | j "d['id']") +smoke "base create (CLI)" $CLI base get "$SB2" +smoke "base delete (CLI)" $CLI base delete "$SB2" smoke "auth token list" $CLI auth token list smoke "base list" $CLI base list smoke "base get" $CLI base get "$BASE" @@ -176,13 +180,46 @@ smoke "plugin list" $CLI plugin list smoke "plugin installs" $CLI plugin installs smoke "plugin csp list" $CLI plugin csp list -# ---------- auth token create/delete (self-cleaning PAT) ---------- -PATID=$($CLI auth token create --name cli-smoke --json 2>/dev/null | j "d['id']") || PATID="" -if [ -n "${PATID:-}" ]; then - smoke "auth token create" $CLI auth token list - smoke "auth token delete" $CLI auth token delete "$PATID" +# ---------- auth token routes (via low-priv account session; PAT cannot mint PATs) ---------- +# Provide SMOKE_LOWPRIV_EMAIL / SMOKE_LOWPRIV_PASSWORD (any account you may sign in). +if [ -n "${SMOKE_LOWPRIV_EMAIL:-}" ] && [ -n "${SMOKE_LOWPRIV_PASSWORD:-}" ]; then + curl -s -c /tmp/cli-smoke-cookie.txt "$EP/api/auth/signin" -X POST -H 'Content-Type: application/json' \ + -d "{\"email\":\"$SMOKE_LOWPRIV_EMAIL\",\"password\":\"$SMOKE_LOWPRIV_PASSWORD\"}" >/dev/null + PATID=$(curl -s -b /tmp/cli-smoke-cookie.txt -X POST "$EP/api/access-token" -H 'Content-Type: application/json' \ + -d '{"name":"cli-smoke","scopes":["table|read"],"expiredTime":"2027-01-01T00:00:00.000Z"}' \ + | python3 -c "import sys,json;print(json.load(sys.stdin).get('id',''))") + if [ -n "$PATID" ]; then + CODE=$(curl -s -o /dev/null -w '%{http_code}' -b /tmp/cli-smoke-cookie.txt -X DELETE "$EP/api/access-token/$PATID") + if [ "$CODE" = "200" ]; then PASS=$((PASS+1)); echo "pass auth token create+delete (REST, session)"; else FAIL=$((FAIL+1)); echo "FAIL auth token delete (HTTP $CODE)"; fi + else + echo "warn auth token create via session failed — check creds" + fi else - echo "warn auth token create returned no id (may need scopes) — check manually" + echo "warn auth-token routes not covered (set SMOKE_LOWPRIV_EMAIL/PASSWORD to enable)" +fi + +# ---------- plugin chain (opt-in: SMOKE_PLUGINS=1; uploads to the registry, cleaned after) ---------- +if [ "${SMOKE_PLUGINS:-0}" = "1" ] && [ -f /root/pi-skill-lab/plugin-workspace/fixture-deploy-W/plugin.json ]; then + FIXT=/root/pi-skill-lab/plugin-workspace/fixture-deploy-W + PID2=$(python3 -c "import json;print(json.load(open('$FIXT/plugin.json'))['id'])") + UP=$(curl -s -b "auth_session=unused" -H "Authorization: Bearer $TOK" \ + -F "js=@$FIXT/dist/main.js" -F "manifest=@$FIXT/plugin.json" \ + "$EP/api/plugin/$PID2/upload-bundle" | python3 -c "import sys,json;print(json.load(sys.stdin).get('id',''))" 2>/dev/null) + if [ -n "$UP" ]; then + smoke "plugin upload-bundle" $CLI plugin list + smoke "plugin-config enable" curl -sf -X PUT -H "Authorization: Bearer $TOK" -H 'Content-Type: application/json' \ + -d '{"enabled":true,"tableIds":[]}' "$EP/api/base/$BASE/plugin-config/$PID2" -o /dev/null + smoke "plugin install" $CLI plugin install --plugin "$PID2" --name cli-smoke --position base + IID=$($CLI plugin installs --json 2>/dev/null | python3 -c " +import sys,json +for i in (json.load(sys.stdin) or []): + if i.get('pluginId')=='$PID2': print(i.get('id') or i.get('installId','')); break" 2>/dev/null) + [ -n "$IID" ] && smoke "plugin uninstall" $CLI plugin uninstall --install "$IID" + CODE=$(curl -s -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $TOK" "$EP/api/plugin/$PID2") + if [ "$CODE" = "200" ]; then PASS=$((PASS+1)); echo "pass plugin registry cleanup"; else FAIL=$((FAIL+1)); echo "FAIL plugin delete (HTTP $CODE)"; fi + else + echo "warn plugin upload failed — PAT lacks register rights: POST /api/plugin and DELETE /api/plugin/:id are admin-only (403); upload-bundle alone needs the plugin_developer whitelist. Provide an admin session to cover the full chain." + fi fi # ---------- table delete (destructive, scratch only) ---------- diff --git a/skills/inkreach-teable-plugin/dynamic-plugins/deployment.md b/skills/inkreach-teable-plugin/dynamic-plugins/deployment.md index 05ad2e4..27eaa04 100644 --- a/skills/inkreach-teable-plugin/dynamic-plugins/deployment.md +++ b/skills/inkreach-teable-plugin/dynamic-plugins/deployment.md @@ -29,6 +29,12 @@ Tokens (`Authorization: Bearer `). **Returns** `201` with `{ id, name, version, ... }` — the plugin ID to use for enable and install. +**Registering a NEW plugin id**: `POST /api/plugin` (manifest as JSON body) — +**admin-only** (PAT gets 403 `restricted_resource`). Uploading a bundle for an +already-registered id only needs the `plugin_developer` whitelist. Deleting a +plugin (`DELETE /api/plugin/:id`) is likewise admin-only. PAT-only automation +should reuse existing registered ids, or ask an admin to register/delete. + **Security** (7-layer validation): 1. Authentication — valid session cookie required (401 if missing) 2. Developer whitelist — `plugin_developer` table lookup (403 if not whitelisted) diff --git a/skills/inkreach-teable/SKILL.md b/skills/inkreach-teable/SKILL.md index dd55389..fe98fb9 100644 --- a/skills/inkreach-teable/SKILL.md +++ b/skills/inkreach-teable/SKILL.md @@ -9,6 +9,13 @@ description: >- Operate a deployed inkreach-teable instance via the `inkreach-teable` CLI. Respond in the user's language. Keep answers concise and action-oriented. +## 0. 执行守则 + +- 多步任务先规划再动手;**完成全部步骤后才给最终答复**,不要中途停止或只给命令不给结果。 +- 完成后如实汇报:做了什么、关键命令、结果;**做不到就明说做不到及原因,绝不谎报成功**。 +- 长任务(批量记录、多工作流)分批执行,每批确认后再继续。 +- **改错了怎么撤销**:workflow 的 snapshot 仅供审计,**没有恢复 API**——用 `workflow deactivate` 止损,再用 `trigger set` / `action update` 改回(节点更新是全图替换,直接改回正确配置即可);整条不要了用 `workflow delete`(软删)。 + > **Critical**: the automation engine is **self-built and differs from official > teable** — the official `teable` CLI corrupts or 404s on automations here. > Exact key-name differences are in [Key Constraints](#3-key-constraints). diff --git a/skills/inkreach-teable/guides/cli-reference.md b/skills/inkreach-teable/guides/cli-reference.md index acaa129..ea68053 100644 --- a/skills/inkreach-teable/guides/cli-reference.md +++ b/skills/inkreach-teable/guides/cli-reference.md @@ -27,6 +27,15 @@ inkreach-teable auth token delete - `auth token create` returns the full token **only once**. Default scopes: `automation|read,automation|create,automation|update,automation|delete,table|read,record|read,record|create,record|update`. +## space / base + +```bash +inkreach-teable space list +inkreach-teable space create --name "" +inkreach-teable base create --space-id --name "" +inkreach-teable base delete # DESTRUCTIVE: soft-deletes all tables in it +``` + ## base / table / field / record / view These resources use the official-teable-compatible API. Full guide: [tables-fields.md](tables-fields.md). diff --git a/src/commands/resources.ts b/src/commands/resources.ts index 25d3055..15d177f 100644 --- a/src/commands/resources.ts +++ b/src/commands/resources.ts @@ -50,7 +50,7 @@ async function fieldNameMap(client: Client, tableId: string): Promise', 'Space id (see `space list`)') + .requiredOption('--name ', 'New base name') + .action( + run(async (ctx, _a, o) => { + const opts = o as { spaceId: string; name: string }; + const base = await ctx.client.createBase(opts.spaceId, opts.name); + if (ctx.json) printJson(base); + else { + const b = base as Record; + printKv([['id', b.id], ['name', b.name]]); + } + }) + ); + + cmd + .command('delete ') + .description('Soft-delete a base (DESTRUCTIVE: hides all its tables)') + .action( + run(async (ctx, args) => { + const baseId = args[0] as string; + await ctx.client.deleteBase(baseId); + if (ctx.json) printJson({ ok: true, id: baseId }); + else process.stdout.write(`✓ Base ${baseId} deleted (soft)\n`); + }) + ); + cmd .command('get ') .description('Get base details') @@ -100,6 +129,54 @@ function baseCmd(): Command { return cmd; } +// ===================== space ===================== + +function spaceCmd(): Command { + const cmd = new Command('space'); + cmd.description('Manage spaces (top-level containers for bases)'); + + cmd + .command('list') + .description('List spaces you can access') + .action( + run(async (ctx) => { + const list = (await ctx.client.listSpaces()) as Array>; + if (ctx.json) printJson(list); + else { + if (!list.length) { + process.stdout.write('No spaces found.\n'); + return; + } + printTable( + [ + { key: 'id', header: 'ID' }, + { key: 'name', header: 'NAME' }, + ], + list.map((sp) => ({ id: truncate(sp.id), name: truncate(sp.name, 40) })) + ); + } + }) + ); + + cmd + .command('create') + .description('Create a space') + .requiredOption('--name ', 'New space name') + .action( + run(async (ctx, _a, o) => { + const opts = o as { name: string }; + const sp = await ctx.client.createSpace(opts.name); + if (ctx.json) printJson(sp); + else { + const s2 = sp as Record; + printKv([['id', s2.id], ['name', s2.name]]); + } + }) + ); + + return cmd; +} + // ===================== table ===================== function tableCmd(): Command { diff --git a/src/lib/client.ts b/src/lib/client.ts index 0061141..035df30 100644 --- a/src/lib/client.ts +++ b/src/lib/client.ts @@ -54,6 +54,18 @@ export class Client { getBase(baseId: string) { return this.call('GET', `/api/base/${baseId}`); } + createBase(spaceId: string, name: string) { + return this.call('POST', '/api/base', { spaceId, name }); + } + deleteBase(baseId: string) { + return this.call<{ id: string }>('DELETE', `/api/base/${baseId}`); + } + listSpaces() { + return this.call('GET', '/api/space'); + } + createSpace(name: string) { + return this.call('POST', '/api/space', { name }); + } // ---- table ---- listTables(baseId: string) { diff --git a/src/lib/ctx.ts b/src/lib/ctx.ts index 9aacc3d..391d0f7 100644 --- a/src/lib/ctx.ts +++ b/src/lib/ctx.ts @@ -28,6 +28,10 @@ export function buildHint(status: number, message: string): string | undefined { return '违反了字段的 visibilityFilter（行级条件）—— `inkreach-teable field visible-choices ` 查看该行可见选项。'; if (/expected array, received/i.test(m)) return '参数形状不符 —— 批量接口要数组；单条可改用 --fields，批量用 --records。'; + if (status === 429) + return '触发限流 —— 稍等后重试；批量操作请降低并发或分批执行。'; + if (/timeout|ETIMEDOUT|ECONNRESET/i.test(m)) + return '请求超时/连接中断 —— 先用只读命令(如 table list)确认服务可达,再重试原操作。'; if (/VALIDATION|Validation error/i.test(m) && status === 400) return '请求校验失败 —— 用 --json 查看完整错误；对照 ` --help` 的参数示例。'; return undefined;