feat(cli): base/space 命令 + 执行守则 + eval harness 入仓 + 多轮 eval (0.1.9)

- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖)
- SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引
  (snapshot 无恢复 API,用 deactivate+节点改回)
- hint 补 429 限流与超时模式
- evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/
  EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分
- 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用
- 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次;
  多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话)
- 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
This commit is contained in:
yeuimu
2026-09-30 08:12:16 +00:00
parent 517dde9dee
commit c9694511a1
20 changed files with 1607 additions and 9 deletions
+44 -7
View File
@@ -51,8 +51,12 @@ cleanup() { curl -s -o /dev/null -X DELETE -H "Authorization: Bearer $TOK" "$EP/
trap cleanup EXIT
echo "sandbox base: $BASE"
# ---------- auth / base ----------
# ---------- auth / base / space ----------
smoke "auth status" $CLI auth status
smoke "space list" $CLI space list
SB2=$($CLI base create --space-id "$SPACE" --name cli-smoke-b2 --json | j "d['id']")
smoke "base create (CLI)" $CLI base get "$SB2"
smoke "base delete (CLI)" $CLI base delete "$SB2"
smoke "auth token list" $CLI auth token list
smoke "base list" $CLI base list
smoke "base get" $CLI base get "$BASE"
@@ -176,13 +180,46 @@ smoke "plugin list" $CLI plugin list
smoke "plugin installs" $CLI plugin installs
smoke "plugin csp list" $CLI plugin csp list
# ---------- auth token create/delete (self-cleaning PAT) ----------
PATID=$($CLI auth token create --name cli-smoke --json 2>/dev/null | j "d['id']") || PATID=""
if [ -n "${PATID:-}" ]; then
smoke "auth token create" $CLI auth token list
smoke "auth token delete" $CLI auth token delete "$PATID"
# ---------- auth token routes (via low-priv account session; PAT cannot mint PATs) ----------
# Provide SMOKE_LOWPRIV_EMAIL / SMOKE_LOWPRIV_PASSWORD (any account you may sign in).
if [ -n "${SMOKE_LOWPRIV_EMAIL:-}" ] && [ -n "${SMOKE_LOWPRIV_PASSWORD:-}" ]; then
curl -s -c /tmp/cli-smoke-cookie.txt "$EP/api/auth/signin" -X POST -H 'Content-Type: application/json' \
-d "{\"email\":\"$SMOKE_LOWPRIV_EMAIL\",\"password\":\"$SMOKE_LOWPRIV_PASSWORD\"}" >/dev/null
PATID=$(curl -s -b /tmp/cli-smoke-cookie.txt -X POST "$EP/api/access-token" -H 'Content-Type: application/json' \
-d '{"name":"cli-smoke","scopes":["table|read"],"expiredTime":"2027-01-01T00:00:00.000Z"}' \
| python3 -c "import sys,json;print(json.load(sys.stdin).get('id',''))")
if [ -n "$PATID" ]; then
CODE=$(curl -s -o /dev/null -w '%{http_code}' -b /tmp/cli-smoke-cookie.txt -X DELETE "$EP/api/access-token/$PATID")
if [ "$CODE" = "200" ]; then PASS=$((PASS+1)); echo "pass auth token create+delete (REST, session)"; else FAIL=$((FAIL+1)); echo "FAIL auth token delete (HTTP $CODE)"; fi
else
echo "warn auth token create via session failed — check creds"
fi
else
echo "warn auth token create returned no id (may need scopes) — check manually"
echo "warn auth-token routes not covered (set SMOKE_LOWPRIV_EMAIL/PASSWORD to enable)"
fi
# ---------- plugin chain (opt-in: SMOKE_PLUGINS=1; uploads to the registry, cleaned after) ----------
if [ "${SMOKE_PLUGINS:-0}" = "1" ] && [ -f /root/pi-skill-lab/plugin-workspace/fixture-deploy-W/plugin.json ]; then
FIXT=/root/pi-skill-lab/plugin-workspace/fixture-deploy-W
PID2=$(python3 -c "import json;print(json.load(open('$FIXT/plugin.json'))['id'])")
UP=$(curl -s -b "auth_session=unused" -H "Authorization: Bearer $TOK" \
-F "js=@$FIXT/dist/main.js" -F "manifest=@$FIXT/plugin.json" \
"$EP/api/plugin/$PID2/upload-bundle" | python3 -c "import sys,json;print(json.load(sys.stdin).get('id',''))" 2>/dev/null)
if [ -n "$UP" ]; then
smoke "plugin upload-bundle" $CLI plugin list
smoke "plugin-config enable" curl -sf -X PUT -H "Authorization: Bearer $TOK" -H 'Content-Type: application/json' \
-d '{"enabled":true,"tableIds":[]}' "$EP/api/base/$BASE/plugin-config/$PID2" -o /dev/null
smoke "plugin install" $CLI plugin install --plugin "$PID2" --name cli-smoke --position base
IID=$($CLI plugin installs --json 2>/dev/null | python3 -c "
import sys,json
for i in (json.load(sys.stdin) or []):
if i.get('pluginId')=='$PID2': print(i.get('id') or i.get('installId','')); break" 2>/dev/null)
[ -n "$IID" ] && smoke "plugin uninstall" $CLI plugin uninstall --install "$IID"
CODE=$(curl -s -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $TOK" "$EP/api/plugin/$PID2")
if [ "$CODE" = "200" ]; then PASS=$((PASS+1)); echo "pass plugin registry cleanup"; else FAIL=$((FAIL+1)); echo "FAIL plugin delete (HTTP $CODE)"; fi
else
echo "warn plugin upload failed — PAT lacks register rights: POST /api/plugin and DELETE /api/plugin/:id are admin-only (403); upload-bundle alone needs the plugin_developer whitelist. Provide an admin session to cover the full chain."
fi
fi
# ---------- table delete (destructive, scratch only) ----------