feat(cli): base/space 命令 + 执行守则 + eval harness 入仓 + 多轮 eval (0.1.9)
- 新命令: space list/create、base create/delete(实测通过,冒烟覆盖) - SKILL.md §0 执行守则:完成全部步骤再答复/不谎报/分批/撤销指引 (snapshot 无恢复 API,用 deactivate+节点改回) - hint 补 429 限流与超时模式 - evals/ 入仓: 14+6 题 harness 参数化(EVAL_ROOT/EVAL_ENV/EVAL_MODEL/ EVAL_CFG/auto_skill 触发模式)+ 多轮修正式 eval 与灵活判分 - 判分器加固: workflow list 错误负载不崩、官方 CLI 探测(command -v)不算使用 - 实测: auto 触发批(入仓版重跑) 11/11、99.0% 首试、100% ≤2 次; 多轮修正 2/2+3/3(修改而非重建);冒烟 85 用例含 token 路由(低权会话) - 运维事实入档: POST/DELETE /api/plugin 为管理员专属(PAT 403)
This commit is contained in:
@@ -51,8 +51,12 @@ cleanup() { curl -s -o /dev/null -X DELETE -H "Authorization: Bearer $TOK" "$EP/
|
||||
trap cleanup EXIT
|
||||
echo "sandbox base: $BASE"
|
||||
|
||||
# ---------- auth / base ----------
|
||||
# ---------- auth / base / space ----------
|
||||
smoke "auth status" $CLI auth status
|
||||
smoke "space list" $CLI space list
|
||||
SB2=$($CLI base create --space-id "$SPACE" --name cli-smoke-b2 --json | j "d['id']")
|
||||
smoke "base create (CLI)" $CLI base get "$SB2"
|
||||
smoke "base delete (CLI)" $CLI base delete "$SB2"
|
||||
smoke "auth token list" $CLI auth token list
|
||||
smoke "base list" $CLI base list
|
||||
smoke "base get" $CLI base get "$BASE"
|
||||
@@ -176,13 +180,46 @@ smoke "plugin list" $CLI plugin list
|
||||
smoke "plugin installs" $CLI plugin installs
|
||||
smoke "plugin csp list" $CLI plugin csp list
|
||||
|
||||
# ---------- auth token create/delete (self-cleaning PAT) ----------
|
||||
PATID=$($CLI auth token create --name cli-smoke --json 2>/dev/null | j "d['id']") || PATID=""
|
||||
if [ -n "${PATID:-}" ]; then
|
||||
smoke "auth token create" $CLI auth token list
|
||||
smoke "auth token delete" $CLI auth token delete "$PATID"
|
||||
# ---------- auth token routes (via low-priv account session; PAT cannot mint PATs) ----------
|
||||
# Provide SMOKE_LOWPRIV_EMAIL / SMOKE_LOWPRIV_PASSWORD (any account you may sign in).
|
||||
if [ -n "${SMOKE_LOWPRIV_EMAIL:-}" ] && [ -n "${SMOKE_LOWPRIV_PASSWORD:-}" ]; then
|
||||
curl -s -c /tmp/cli-smoke-cookie.txt "$EP/api/auth/signin" -X POST -H 'Content-Type: application/json' \
|
||||
-d "{\"email\":\"$SMOKE_LOWPRIV_EMAIL\",\"password\":\"$SMOKE_LOWPRIV_PASSWORD\"}" >/dev/null
|
||||
PATID=$(curl -s -b /tmp/cli-smoke-cookie.txt -X POST "$EP/api/access-token" -H 'Content-Type: application/json' \
|
||||
-d '{"name":"cli-smoke","scopes":["table|read"],"expiredTime":"2027-01-01T00:00:00.000Z"}' \
|
||||
| python3 -c "import sys,json;print(json.load(sys.stdin).get('id',''))")
|
||||
if [ -n "$PATID" ]; then
|
||||
CODE=$(curl -s -o /dev/null -w '%{http_code}' -b /tmp/cli-smoke-cookie.txt -X DELETE "$EP/api/access-token/$PATID")
|
||||
if [ "$CODE" = "200" ]; then PASS=$((PASS+1)); echo "pass auth token create+delete (REST, session)"; else FAIL=$((FAIL+1)); echo "FAIL auth token delete (HTTP $CODE)"; fi
|
||||
else
|
||||
echo "warn auth token create via session failed — check creds"
|
||||
fi
|
||||
else
|
||||
echo "warn auth token create returned no id (may need scopes) — check manually"
|
||||
echo "warn auth-token routes not covered (set SMOKE_LOWPRIV_EMAIL/PASSWORD to enable)"
|
||||
fi
|
||||
|
||||
# ---------- plugin chain (opt-in: SMOKE_PLUGINS=1; uploads to the registry, cleaned after) ----------
|
||||
if [ "${SMOKE_PLUGINS:-0}" = "1" ] && [ -f /root/pi-skill-lab/plugin-workspace/fixture-deploy-W/plugin.json ]; then
|
||||
FIXT=/root/pi-skill-lab/plugin-workspace/fixture-deploy-W
|
||||
PID2=$(python3 -c "import json;print(json.load(open('$FIXT/plugin.json'))['id'])")
|
||||
UP=$(curl -s -b "auth_session=unused" -H "Authorization: Bearer $TOK" \
|
||||
-F "js=@$FIXT/dist/main.js" -F "manifest=@$FIXT/plugin.json" \
|
||||
"$EP/api/plugin/$PID2/upload-bundle" | python3 -c "import sys,json;print(json.load(sys.stdin).get('id',''))" 2>/dev/null)
|
||||
if [ -n "$UP" ]; then
|
||||
smoke "plugin upload-bundle" $CLI plugin list
|
||||
smoke "plugin-config enable" curl -sf -X PUT -H "Authorization: Bearer $TOK" -H 'Content-Type: application/json' \
|
||||
-d '{"enabled":true,"tableIds":[]}' "$EP/api/base/$BASE/plugin-config/$PID2" -o /dev/null
|
||||
smoke "plugin install" $CLI plugin install --plugin "$PID2" --name cli-smoke --position base
|
||||
IID=$($CLI plugin installs --json 2>/dev/null | python3 -c "
|
||||
import sys,json
|
||||
for i in (json.load(sys.stdin) or []):
|
||||
if i.get('pluginId')=='$PID2': print(i.get('id') or i.get('installId','')); break" 2>/dev/null)
|
||||
[ -n "$IID" ] && smoke "plugin uninstall" $CLI plugin uninstall --install "$IID"
|
||||
CODE=$(curl -s -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $TOK" "$EP/api/plugin/$PID2")
|
||||
if [ "$CODE" = "200" ]; then PASS=$((PASS+1)); echo "pass plugin registry cleanup"; else FAIL=$((FAIL+1)); echo "FAIL plugin delete (HTTP $CODE)"; fi
|
||||
else
|
||||
echo "warn plugin upload failed — PAT lacks register rights: POST /api/plugin and DELETE /api/plugin/:id are admin-only (403); upload-bundle alone needs the plugin_developer whitelist. Provide an admin session to cover the full chain."
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------- table delete (destructive, scratch only) ----------
|
||||
|
||||
Reference in New Issue
Block a user