feat: 浏览器新增 browser_find_text / browser_click_point——AX 快照够不到的元素也能定位点击

apps/desktop/src/main/browser/browser-controller.ts

@@ -12,6 +12,7 @@ import {
 import {
   assertSafeBrowserUrl,
   browserObservationNameLimit,
+  MAX_BROWSER_FIND_TEXT_RESULTS,
   NAMELESS_ALLOWED_ROLES,
   prioritizeBrowserObservationCandidates,
   resolveBrowserDestination,
@@ -60,6 +61,8 @@ interface TabRecord {
   commandTail: Promise<unknown>
   lastLayoutRevision: number
   attached: boolean
+  /** 最近一次截图的 像素/DIP 比例（browser_click_point 换算截图坐标用） */
+  lastShotScale: number
 }
 
 interface BrowserSettings {
@@ -75,6 +78,23 @@ export interface BrowserObservation {
   elements: Array<{ ref: string; role: string; name: string; editable: boolean }>
 }
 
+/** browser_find_text 的候选描述。rect 为视口 CSS 像素（iframe 内元素已换算为顶层页面坐标） */
+export interface BrowserFindTextCandidate {
+  index: number
+  tagName: string
+  text: string
+  rect: { x: number; y: number; w: number; h: number }
+  context: string
+  /** true = 位于 iframe 内（不能直接注册 ref 点击，需改用坐标点击） */
+  iframe?: boolean
+}
+
+export interface BrowserFindTextResult extends BrowserObservation {
+  matched: number
+  picked?: number
+  candidates?: BrowserFindTextCandidate[]
+}
+
 export interface BrowserWaitCondition {
   kind: 'url' | 'text' | 'selector'
   value: string
@@ -119,6 +139,140 @@ const KEY_EVENTS: Record<string, { key: string; code: string; vk: number }> = {
   space: { key: ' ', code: 'Space', vk: 32 }
 }
 
+/** 页面内文本/选择器查找器（宿主固定函数，参数经 JSON 注入，非任意脚本）。
+ * 遍历主文档 + 开放 shadow DOM + 同源 iframe（深度受限），按元素自身直接可见文本或
+ * CSS 选择器匹配。返回纯可序列化数据（results 元数据 + diag 诊断 + pick 模式打标结果）。
+ * 背景：纯 div/span + JS 点击的元素（商品规格等）不在 AX 快照里，ref 体系够不到它们。
+ * 注意：结果回传只用 returnByValue 序列化——Runtime.callFunctionOn 跨命令持有
+ * RemoteObject 引用在 Electron debugger 下不可靠（this 绑定丢失，恒返回 undefined）。 */
+const FIND_TEXT_FINDER = String.raw`function(input) {
+  const norm = (s) => (s || '').replace(/\s+/g, ' ').trim()
+  const results = []
+  const docs = []
+  let visitedTotal = 0
+  const collectDocs = (doc, depth) => {
+    if (!doc || docs.indexOf(doc) >= 0 || depth > 3) return
+    docs.push(doc)
+    let frames = []
+    try { frames = Array.prototype.slice.call(doc.querySelectorAll('iframe')) } catch (err) {}
+    for (const frame of frames) {
+      try { collectDocs(frame.contentDocument, depth + 1) } catch (err) {}
+    }
+  }
+  collectDocs(document, 0)
+  if (input.selector) {
+    try { document.querySelector(input.selector) } catch (err) {
+      return { error: '选择器语法非法' }
+    }
+  }
+  const ownText = (el) => {
+    let text = ''
+    for (let node = el.firstChild; node; node = node.nextSibling) {
+      if (node.nodeType === 3) text += node.nodeValue
+    }
+    return norm(text)
+  }
+  const contextOf = (el) => {
+    let parent = el.parentElement
+    while (parent) {
+      const text = norm(parent.textContent)
+      if (text) return text.slice(0, 60)
+      parent = parent.parentElement
+    }
+    return ''
+  }
+  const visible = (el) => {
+    try {
+      const rect = el.getBoundingClientRect()
+      if (rect.width < 1 || rect.height < 1) return false
+      const view = el.ownerDocument.defaultView
+      if (!view) return false
+      const style = view.getComputedStyle(el)
+      return style.visibility !== 'hidden' && style.pointerEvents !== 'none'
+    } catch (err) {
+      return false
+    }
+  }
+  const toTopCoords = (el) => {
+    let doc = el.ownerDocument
+    let x = 0
+    let y = 0
+    let guard = 0
+    while (doc && doc !== document && guard++ < 5) {
+      const findHost = (d) => {
+        let frames = []
+        try { frames = Array.prototype.slice.call(d.querySelectorAll('iframe')) } catch (err) {}
+        for (const frame of frames) {
+          if (frame.contentDocument === doc) return frame
+          const nested = findHost(frame.contentDocument)
+          if (nested) return nested
+        }
+        return null
+      }
+      const host = findHost(document)
+      if (!host) break
+      const hostRect = host.getBoundingClientRect()
+      x += hostRect.x
+      y += hostRect.y
+      doc = host.ownerDocument
+    }
+    return { x, y }
+  }
+  const hits = []
+  const walk = (root) => {
+    let iter = null
+    try { iter = root.createTreeWalker(root, NodeFilter.SHOW_ELEMENT) } catch (err) { return }
+    let visited = 0
+    let node
+    while ((node = iter.nextNode())) {
+      if (++visited > 20000) break
+      visitedTotal++
+      const el = node
+      if (el.shadowRoot) walk(el.shadowRoot)
+      let hit = false
+      let label = ''
+      if (input.selector) {
+        try { hit = el.matches(input.selector) } catch (err) { hit = false }
+        if (hit) {
+          label = ownText(el)
+          if (!label) {
+            try { label = norm(el.getAttribute('aria-label') || el.getAttribute('title') || '') } catch (err) {}
+          }
+        }
+      } else {
+        label = ownText(el)
+        if (label) hit = input.contains ? label.indexOf(input.text) >= 0 : label === input.text
+      }
+      if (hit && visible(el)) hits.push({ el, label })
+    }
+  }
+  for (const doc of docs) {
+    if (doc.documentElement) walk(doc)
+  }
+  if (input.pickIndex !== undefined && input.pickIndex !== null) {
+    const hit = hits[input.pickIndex - 1]
+    if (!hit) return { ok: false, diag: { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href) } }
+    hit.el.setAttribute('data-aso-find', input.tagValue)
+    const rect = hit.el.getBoundingClientRect()
+    const offset = hit.el.ownerDocument === document ? { x: 0, y: 0 } : toTopCoords(hit.el)
+    return { ok: true, tagged: hit.label.slice(0, 80), rect: { x: Math.round(rect.x + offset.x), y: Math.round(rect.y + offset.y) }, diag: { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href) } }
+  }
+  for (const hit of hits.slice(0, input.limit)) {
+    const rect = hit.el.getBoundingClientRect()
+    const offset = hit.el.ownerDocument === document ? { x: 0, y: 0 } : toTopCoords(hit.el)
+    results.push({
+      index: results.length + 1,
+      tagName: hit.el.tagName.toLowerCase(),
+      text: hit.label.slice(0, 80),
+      rect: { x: Math.round(rect.x + offset.x), y: Math.round(rect.y + offset.y), w: Math.round(rect.width), h: Math.round(rect.height) },
+      context: contextOf(hit.el),
+      iframe: hit.el.ownerDocument !== document
+    })
+  }
+  const diag = { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href), readyState: document.readyState, nodeFilter: typeof NodeFilter }
+  return { results, diag }
+}`
+
 export class BrowserController {
   private owner: BrowserWindow | null = null
   private browserSession: Session
@@ -321,7 +475,8 @@ export class BrowserController {
       generation: 0,
       commandTail: Promise.resolve(),
       lastLayoutRevision: 0,
-      attached: false
+      attached: false,
+      lastShotScale: 1
     }
     view.setVisible(false)
 
@@ -712,6 +867,92 @@ export class BrowserController {
     })
   }
 
+  /** 按可见文本/CSS 选择器在 DOM 中直查（不依赖无障碍树）：命中后注册为普通 ref，
+   * browser_click 等后续操作照常使用。多命中且未指定 index 时只返回候选列表不发 ref；
+   * 仅在真正注册 ref 时才作废旧 ref（与 observe/find 的失效语义一致）。 */
+  async findText(
+    query: { text?: string; selector?: string; contains?: boolean; index?: number },
+    tabId?: string
+  ): Promise<BrowserFindTextResult> {
+    const text = query.text?.trim().slice(0, 200) ?? ''
+    const selector = query.selector?.trim().slice(0, 500) ?? ''
+    if ((text ? 1 : 0) + (selector ? 1 : 0) !== 1) throw new Error('text 与 selector 必须且只能提供一个。')
+    this.assertRiskAcknowledged()
+    const tab = this.getAgentTab(tabId)
+    this.agentTabId = tab.tabId
+    return this.runAgentOp(tab, async () => {
+      const baseInput = { text: text || undefined, selector: selector || undefined, contains: query.contains === true, limit: MAX_BROWSER_FIND_TEXT_RESULTS }
+      const payload = await this.evaluateOnPage(tab, `((${FIND_TEXT_FINDER})(${JSON.stringify(baseInput)}))`) as
+        | { error?: string; results?: unknown[]; diag?: unknown }
+        | null
+      if (!payload || typeof payload !== 'object') throw new Error('页面定位结果异常，请重试。')
+      if (payload.error) throw new Error(`定位失败：${payload.error}`)
+      const diag = payload.diag
+      const candidates = (Array.isArray(payload.results) ? payload.results : []).filter(
+        (c): c is BrowserFindTextCandidate =>
+          !!c && typeof c === 'object' && typeof (c as BrowserFindTextCandidate).tagName === 'string'
+      )
+      if (candidates.length === 0) {
+        this.trace(tab, 'find_text', `未找到 ${text || selector}`)
+        return {
+          tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation,
+          matched: 0, elements: [], candidates: [], ...(diag !== undefined ? { diag } : {})
+        }
+      }
+      const picked = query.index ?? 1
+      if (!Number.isInteger(picked) || picked < 1 || picked > candidates.length) {
+        throw new Error(`index 必须是 1 到 ${candidates.length} 之间的整数。`)
+      }
+      if (candidates.length > 1 && query.index === undefined) {
+        this.trace(tab, 'find_text', `「${text || selector}」命中 ${candidates.length} 处，等待指定 index`)
+        return {
+          tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation,
+          matched: candidates.length, elements: [], candidates, ...(diag !== undefined ? { diag } : {})
+        }
+      }
+      const candidate = candidates[picked - 1]
+      if (candidate.iframe === true) {
+        throw new Error('目标元素位于 iframe 内，无法直接注册 ref：请基于 browser_screenshot 用 browser_click_point 点击（候选列表 rect 已换算为页面顶层坐标）。')
+      }
+      // 选定阶段：给第 N 个命中打临时 data 属性，再经 DOM 域换成 backendNodeId。
+      // （DOM.querySelector 不穿 shadow DOM，shadow 内的命中会在下一步报"未找到"，
+      //  报错信息会引导改用坐标点击）
+      const tagValue = Math.random().toString(36).slice(2, 10)
+      const pickPayload = await this.evaluateOnPage(
+        tab,
+        `((${FIND_TEXT_FINDER})(${JSON.stringify({ ...baseInput, pickIndex: picked, tagValue })}))`
+      ) as { ok?: boolean; tagged?: string } | null
+      if (!pickPayload || pickPayload.ok !== true) {
+        throw new Error('页面已变化，目标元素不存在了，请重新调用 browser_find_text。')
+      }
+      const documentResponse = await this.cdp(tab, 'DOM.getDocument', { depth: 0 })
+      const rootId = (documentResponse.root as { nodeId?: number } | undefined)?.nodeId
+      if (typeof rootId !== 'number' || rootId <= 0) throw new Error('无法访问页面文档，请重试。')
+      const queryResponse = await this.cdp(tab, 'DOM.querySelector', {
+        nodeId: rootId,
+        selector: `[data-aso-find="${tagValue}"]`
+      })
+      const taggedNodeId = (queryResponse as { nodeId?: number }).nodeId
+      if (typeof taggedNodeId !== 'number' || taggedNodeId === 0) {
+        throw new Error('目标元素无法转换为可点击引用（可能位于 shadow DOM 内），请改用 browser_screenshot + browser_click_point。')
+      }
+      const describeResponse = await this.cdp(tab, 'DOM.describeNode', { nodeId: taggedNodeId })
+      const backendNodeId = (describeResponse.node as { backendNodeId?: number } | undefined)?.backendNodeId
+      await this.cdp(tab, 'DOM.removeAttribute', { nodeId: taggedNodeId, name: 'data-aso-find' }).catch(() => undefined)
+      if (typeof backendNodeId !== 'number' || backendNodeId <= 0) throw new Error('无法解析目标元素，请重新定位。')
+      const elements = this.buildRefs(tab, [
+        { backendNodeId, role: candidate.tagName, name: candidate.text, editable: false }
+      ])
+      this.trace(tab, 'find_text', `定位「${(candidate.text || candidate.tagName).slice(0, 40)}」→ ${elements[0]?.ref}（共 ${candidates.length} 处）`)
+      return {
+        tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation,
+        matched: candidates.length, picked,
+        elements,
+        candidates: candidates.length > 1 ? candidates : undefined
+      }
+    })
+  }
+
   private async clickRef(tab: TabRecord, ref: string, generation: number): Promise<RefEntry> {
     const target = this.resolveRef(tab, ref)
     const { x, y } = await this.centerForRef(tab, ref, generation)
@@ -762,6 +1003,40 @@ export class BrowserController {
     })
   }
 
+  /** 截图坐标兜底点击：坐标以最近一次 browser_screenshot 图像的像素为准（按截图比例换算 DIP）。
+   * 仅供 observe/find/find_text 都无法定位时使用（canvas、封闭 shadow DOM、混淆页面）。 */
+  async clickPoint(x: number, y: number, tabId?: string): Promise<BrowserUiState> {
+    if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0) throw new Error('坐标必须是非负数。')
+    this.assertRiskAcknowledged()
+    const tab = this.getAgentTab(tabId)
+    this.agentTabId = tab.tabId
+    return this.runAgentOp(tab, async () => {
+      const bounds = tab.view.getBounds()
+      if (bounds.width <= 0 || bounds.height <= 0) throw new Error('浏览器面板当前不可见，无法按坐标点击。')
+      const scale = tab.lastShotScale > 0 ? tab.lastShotScale : 1
+      const dipX = x / scale
+      const dipY = y / scale
+      if (dipX >= bounds.width || dipY >= bounds.height) {
+        throw new Error(
+          `坐标越界：换算后 (${Math.round(dipX)}, ${Math.round(dipY)}) 超出页面 ${Math.round(bounds.width)}×${Math.round(bounds.height)}，请基于最新截图重新取坐标。`
+        )
+      }
+      // 点击位置标记（宿主固定脚本，纯视觉反馈）
+      const marker =
+        '((p)=>{try{const d=document.createElement("div");d.style.cssText="position:fixed;left:"+(p.x-9)+"px;top:"+(p.y-9)+' +
+        '"px;width:18px;height:18px;border:2px solid #f59e0b;border-radius:50%;pointer-events:none;z-index:2147483647;box-sizing:border-box";' +
+        'document.documentElement.appendChild(d);setTimeout(()=>d.remove(),700);return true}catch(e){return false}})(' +
+        JSON.stringify({ x: dipX, y: dipY }) +
+        ')'
+      await this.cdp(tab, 'Runtime.evaluate', { expression: marker, returnByValue: true }).catch(() => undefined)
+      await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: dipX, y: dipY })
+      await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: dipX, y: dipY, button: 'left', clickCount: 1 })
+      await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: dipX, y: dipY, button: 'left', clickCount: 1 })
+      this.trace(tab, 'click_point', `点击截图坐标 (${Math.round(x)}, ${Math.round(y)})`)
+      return this.buildState()
+    })
+  }
+
   async fill(ref: string, text: string, tabId?: string): Promise<BrowserUiState> {
     if (text.length > MAX_FILL_CHARS) throw new Error(`单次输入不能超过 ${MAX_FILL_CHARS} 个字符。`)
     this.assertRiskAcknowledged()
@@ -930,6 +1205,9 @@ export class BrowserController {
     const tab = this.getAgentTab(tabId)
     return this.runAgentOp(tab, async () => {
       const image = await tab.view.webContents.capturePage()
+      // 记录 像素/DIP 比例，browser_click_point 换算截图坐标用（capturePage 按显示 DPR 放大）
+      const bounds = tab.view.getBounds()
+      if (bounds.width > 0) tab.lastShotScale = image.getSize().width / bounds.width
       this.trace(tab, 'screenshot', '截取当前页面')
       return { tabId: tab.tabId, url: tab.url, title: tab.title, dataUrl: image.toDataURL() }
     })

apps/desktop/src/main/browser/browser-policy.ts

@@ -78,6 +78,9 @@ export const INTERACTIVE_AX_ROLES = new Set([
 /** 无名称也值得返回的 role（依赖 role 本身可定位） */
 export const NAMELESS_ALLOWED_ROLES = new Set(['button', 'textfield', 'textbox', 'link', 'checkbox', 'combobox'])
 
+/** browser_find_text 单次返回的候选上限（DOM 直查不受快照配额约束，巨页防爆量） */
+export const MAX_BROWSER_FIND_TEXT_RESULTS = 40
+
 export function resolveBrowserObserveMaxElements(requested?: number): number {
   if (typeof requested !== 'number' || !Number.isFinite(requested)) return DEFAULT_BROWSER_OBSERVE_MAX_ELEMENTS
   return Math.min(MAX_BROWSER_OBSERVE_MAX_ELEMENTS, Math.max(MIN_BROWSER_OBSERVE_MAX_ELEMENTS, Math.round(requested)))

apps/desktop/src/main/tools/browser-tools.ts

@@ -8,7 +8,10 @@ import { getBrowserController } from '../browser/browser-controller'
  * 使用范式（与 Playwright MCP / Proma 一致的 AX 快照 + ref 体系）：
  * 1. browser_observe / browser_find 拿到带 ref 的可交互元素快照；
  * 2. 用 ref 调 click / fill / act 等操作；页面变化后旧 ref 失效，必须重新观察；
- * 3. 语义结构足够时优先 observe，需要视觉验证时才 browser_screenshot。
+ * 3. 语义结构足够时优先 observe，需要视觉验证时才 browser_screenshot；
+ * 4. 目标不在 AX 快照里（纯 div/span + JS 点击的元素，如商品规格/容量按钮）时，
+ *    用 browser_find_text 按可见文本或 CSS 选择器在 DOM 中定位，拿到 ref 后照常操作；
+ * 5. 一切定位手段都失效（canvas、封闭 shadow DOM）才 browser_screenshot + browser_click_point。
  */
 
 const TAB_ID_PARAM = Type.Optional(
@@ -72,12 +75,34 @@ export const browserTools = [
     }
   }),
 
+  defineTool({
+    name: 'browser_find_text',
+    label: '按文本定位元素',
+    description:
+      '按可见文本或 CSS 选择器在页面 DOM 中定位元素并返回新 ref（text 与 selector 二选一）。不依赖无障碍树，专治 observe/find 快照里缺失的纯 div/span 可点击元素（商品规格、容量按钮、自定义控件）。text 匹配元素自身的直接文本（空白归一后精确匹配；contains=true 改为包含）。多个命中时不发 ref，返回候选列表（含 rect/上下文），带 index 重调以选定。命中并注册 ref 时作废旧 ref。支持开放 shadow DOM 与同源 iframe（iframe 内元素只能看候选，点击需改用坐标）。注意：快照配额会挤掉普通文本节点，即使 observe 看似完整，找不到目标时也应优先改用本工具。',
+    promptSnippet: 'browser_find_text: 按文本/选择器定位（AX 快照缺失时用）',
+    parameters: Type.Object({
+      text: Type.Optional(Type.String({ description: '元素自身可见文本，如 "256GB"' })),
+      selector: Type.Optional(Type.String({ description: 'CSS 选择器（文本重复或目标无文本时用）' })),
+      contains: Type.Optional(Type.Boolean({ description: '文本包含匹配（默认精确匹配）' })),
+      index: Type.Optional(Type.Number({ minimum: 1, description: '多个候选时指定第几个（1 起，来自上次返回的候选列表）' })),
+      tabId: TAB_ID_PARAM
+    }),
+    async execute(_toolCallId, params) {
+      const result = await getBrowserController().findText(
+        { text: params.text, selector: params.selector, contains: params.contains, index: params.index },
+        params.tabId
+      )
+      return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result }
+    }
+  }),
+
   defineTool({
     name: 'browser_click',
     label: '点击元素',
-    description: '点击最新 observe/find 结果中的指定 ref。页面会短暂高亮目标。点击导致的重渲染会让旧 ref 失效，操作后需重新观察。',
+    description: '点击最新 observe/find/find_text 结果中的指定 ref。页面会短暂高亮目标。点击导致的重渲染会让旧 ref 失效，操作后需重新观察。',
     promptSnippet: 'browser_click: 点击 ref',
-    parameters: Type.Object({ ref: Type.String({ description: '元素引用（来自 observe/find）' }), tabId: TAB_ID_PARAM }),
+    parameters: Type.Object({ ref: Type.String({ description: '元素引用（来自 observe/find/find_text）' }), tabId: TAB_ID_PARAM }),
     async execute(_toolCallId, params) {
       const state = await getBrowserController().click(params.ref, params.tabId)
       return { content: [{ type: 'text', text: JSON.stringify({ clicked: params.ref, url: state.tabs.find((t) => t.tabId === state.activeTabId)?.url }) }], details: state }
@@ -107,6 +132,23 @@ export const browserTools = [
     }
   }),
 
+  defineTool({
+    name: 'browser_click_point',
+    label: '坐标点击（兜底）',
+    description:
+      '点击最近一次 browser_screenshot 图像上的像素坐标 (x, y)。仅当 observe/find/find_text 都无法定位目标时的最后手段（canvas、封闭 shadow DOM、混淆页面）。点击前必须先截最新图；坐标越界会直接报错。',
+    promptSnippet: 'browser_click_point: 截图坐标兜底点击（最后手段）',
+    parameters: Type.Object({
+      x: Type.Number({ minimum: 0, description: '截图图像上的 x 像素' }),
+      y: Type.Number({ minimum: 0, description: '截图图像上的 y 像素' }),
+      tabId: TAB_ID_PARAM
+    }),
+    async execute(_toolCallId, params) {
+      const state = await getBrowserController().clickPoint(params.x, params.y, params.tabId)
+      return { content: [{ type: 'text', text: JSON.stringify({ clicked: [params.x, params.y], url: state.tabs.find((t) => t.tabId === state.activeTabId)?.url }) }], details: state }
+    }
+  }),
+
   defineTool({
     name: 'browser_fill',
     label: '填写输入框',