From 903d393e0999c1b0b652e6e6c134c766757491d6 Mon Sep 17 00:00:00 2001 From: Rick Ye Date: Thu, 10 Sep 2026 15:38:45 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B5=8F=E8=A7=88=E5=99=A8=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=20browser=5Ffind=5Ftext=20/=20browser=5Fclick=5Fpoint?= =?UTF-8?q?=E2=80=94=E2=80=94AX=20=E5=BF=AB=E7=85=A7=E5=A4=9F=E4=B8=8D?= =?UTF-8?q?=E5=88=B0=E7=9A=84=E5=85=83=E7=B4=A0=E4=B9=9F=E8=83=BD=E5=AE=9A?= =?UTF-8?q?=E4=BD=8D=E7=82=B9=E5=87=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/browser/browser-controller.ts | 280 +++++++++++++++++- .../src/main/browser/browser-policy.ts | 3 + apps/desktop/src/main/tools/browser-tools.ts | 48 ++- 3 files changed, 327 insertions(+), 4 deletions(-) diff --git a/apps/desktop/src/main/browser/browser-controller.ts b/apps/desktop/src/main/browser/browser-controller.ts index 0e754c9..917f556 100644 --- a/apps/desktop/src/main/browser/browser-controller.ts +++ b/apps/desktop/src/main/browser/browser-controller.ts @@ -12,6 +12,7 @@ import { import { assertSafeBrowserUrl, browserObservationNameLimit, + MAX_BROWSER_FIND_TEXT_RESULTS, NAMELESS_ALLOWED_ROLES, prioritizeBrowserObservationCandidates, resolveBrowserDestination, @@ -60,6 +61,8 @@ interface TabRecord { commandTail: Promise lastLayoutRevision: number attached: boolean + /** 最近一次截图的 像素/DIP 比例（browser_click_point 换算截图坐标用） */ + lastShotScale: number } interface BrowserSettings { @@ -75,6 +78,23 @@ export interface BrowserObservation { elements: Array<{ ref: string; role: string; name: string; editable: boolean }> } +/** browser_find_text 的候选描述。rect 为视口 CSS 像素（iframe 内元素已换算为顶层页面坐标） */ +export interface BrowserFindTextCandidate { + index: number + tagName: string + text: string + rect: { x: number; y: number; w: number; h: number } + context: string + /** true = 位于 iframe 内（不能直接注册 ref 点击，需改用坐标点击） */ + iframe?: boolean +} + +export interface BrowserFindTextResult extends BrowserObservation { + matched: number + picked?: number + candidates?: BrowserFindTextCandidate[] +} + export interface BrowserWaitCondition { kind: 'url' | 'text' | 'selector' value: string @@ -119,6 +139,140 @@ const KEY_EVENTS: Record = { space: { key: ' ', code: 'Space', vk: 32 } } +/** 页面内文本/选择器查找器（宿主固定函数，参数经 JSON 注入，非任意脚本）。 + * 遍历主文档 + 开放 shadow DOM + 同源 iframe（深度受限），按元素自身直接可见文本或 + * CSS 选择器匹配。返回纯可序列化数据（results 元数据 + diag 诊断 + pick 模式打标结果）。 + * 背景：纯 div/span + JS 点击的元素（商品规格等）不在 AX 快照里，ref 体系够不到它们。 + * 注意：结果回传只用 returnByValue 序列化——Runtime.callFunctionOn 跨命令持有 + * RemoteObject 引用在 Electron debugger 下不可靠（this 绑定丢失，恒返回 undefined）。 */ +const FIND_TEXT_FINDER = String.raw`function(input) { + const norm = (s) => (s || '').replace(/\s+/g, ' ').trim() + const results = [] + const docs = [] + let visitedTotal = 0 + const collectDocs = (doc, depth) => { + if (!doc || docs.indexOf(doc) >= 0 || depth > 3) return + docs.push(doc) + let frames = [] + try { frames = Array.prototype.slice.call(doc.querySelectorAll('iframe')) } catch (err) {} + for (const frame of frames) { + try { collectDocs(frame.contentDocument, depth + 1) } catch (err) {} + } + } + collectDocs(document, 0) + if (input.selector) { + try { document.querySelector(input.selector) } catch (err) { + return { error: '选择器语法非法' } + } + } + const ownText = (el) => { + let text = '' + for (let node = el.firstChild; node; node = node.nextSibling) { + if (node.nodeType === 3) text += node.nodeValue + } + return norm(text) + } + const contextOf = (el) => { + let parent = el.parentElement + while (parent) { + const text = norm(parent.textContent) + if (text) return text.slice(0, 60) + parent = parent.parentElement + } + return '' + } + const visible = (el) => { + try { + const rect = el.getBoundingClientRect() + if (rect.width < 1 || rect.height < 1) return false + const view = el.ownerDocument.defaultView + if (!view) return false + const style = view.getComputedStyle(el) + return style.visibility !== 'hidden' && style.pointerEvents !== 'none' + } catch (err) { + return false + } + } + const toTopCoords = (el) => { + let doc = el.ownerDocument + let x = 0 + let y = 0 + let guard = 0 + while (doc && doc !== document && guard++ < 5) { + const findHost = (d) => { + let frames = [] + try { frames = Array.prototype.slice.call(d.querySelectorAll('iframe')) } catch (err) {} + for (const frame of frames) { + if (frame.contentDocument === doc) return frame + const nested = findHost(frame.contentDocument) + if (nested) return nested + } + return null + } + const host = findHost(document) + if (!host) break + const hostRect = host.getBoundingClientRect() + x += hostRect.x + y += hostRect.y + doc = host.ownerDocument + } + return { x, y } + } + const hits = [] + const walk = (root) => { + let iter = null + try { iter = root.createTreeWalker(root, NodeFilter.SHOW_ELEMENT) } catch (err) { return } + let visited = 0 + let node + while ((node = iter.nextNode())) { + if (++visited > 20000) break + visitedTotal++ + const el = node + if (el.shadowRoot) walk(el.shadowRoot) + let hit = false + let label = '' + if (input.selector) { + try { hit = el.matches(input.selector) } catch (err) { hit = false } + if (hit) { + label = ownText(el) + if (!label) { + try { label = norm(el.getAttribute('aria-label') || el.getAttribute('title') || '') } catch (err) {} + } + } + } else { + label = ownText(el) + if (label) hit = input.contains ? label.indexOf(input.text) >= 0 : label === input.text + } + if (hit && visible(el)) hits.push({ el, label }) + } + } + for (const doc of docs) { + if (doc.documentElement) walk(doc) + } + if (input.pickIndex !== undefined && input.pickIndex !== null) { + const hit = hits[input.pickIndex - 1] + if (!hit) return { ok: false, diag: { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href) } } + hit.el.setAttribute('data-aso-find', input.tagValue) + const rect = hit.el.getBoundingClientRect() + const offset = hit.el.ownerDocument === document ? { x: 0, y: 0 } : toTopCoords(hit.el) + return { ok: true, tagged: hit.label.slice(0, 80), rect: { x: Math.round(rect.x + offset.x), y: Math.round(rect.y + offset.y) }, diag: { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href) } } + } + for (const hit of hits.slice(0, input.limit)) { + const rect = hit.el.getBoundingClientRect() + const offset = hit.el.ownerDocument === document ? { x: 0, y: 0 } : toTopCoords(hit.el) + results.push({ + index: results.length + 1, + tagName: hit.el.tagName.toLowerCase(), + text: hit.label.slice(0, 80), + rect: { x: Math.round(rect.x + offset.x), y: Math.round(rect.y + offset.y), w: Math.round(rect.width), h: Math.round(rect.height) }, + context: contextOf(hit.el), + iframe: hit.el.ownerDocument !== document + }) + } + const diag = { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href), readyState: document.readyState, nodeFilter: typeof NodeFilter } + return { results, diag } +}` + export class BrowserController { private owner: BrowserWindow | null = null private browserSession: Session @@ -321,7 +475,8 @@ export class BrowserController { generation: 0, commandTail: Promise.resolve(), lastLayoutRevision: 0, - attached: false + attached: false, + lastShotScale: 1 } view.setVisible(false) @@ -712,6 +867,92 @@ export class BrowserController { }) } + /** 按可见文本/CSS 选择器在 DOM 中直查（不依赖无障碍树）：命中后注册为普通 ref， + * browser_click 等后续操作照常使用。多命中且未指定 index 时只返回候选列表不发 ref； + * 仅在真正注册 ref 时才作废旧 ref（与 observe/find 的失效语义一致）。 */ + async findText( + query: { text?: string; selector?: string; contains?: boolean; index?: number }, + tabId?: string + ): Promise { + const text = query.text?.trim().slice(0, 200) ?? '' + const selector = query.selector?.trim().slice(0, 500) ?? '' + if ((text ? 1 : 0) + (selector ? 1 : 0) !== 1) throw new Error('text 与 selector 必须且只能提供一个。') + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + this.agentTabId = tab.tabId + return this.runAgentOp(tab, async () => { + const baseInput = { text: text || undefined, selector: selector || undefined, contains: query.contains === true, limit: MAX_BROWSER_FIND_TEXT_RESULTS } + const payload = await this.evaluateOnPage(tab, `((${FIND_TEXT_FINDER})(${JSON.stringify(baseInput)}))`) as + | { error?: string; results?: unknown[]; diag?: unknown } + | null + if (!payload || typeof payload !== 'object') throw new Error('页面定位结果异常，请重试。') + if (payload.error) throw new Error(`定位失败：${payload.error}`) + const diag = payload.diag + const candidates = (Array.isArray(payload.results) ? payload.results : []).filter( + (c): c is BrowserFindTextCandidate => + !!c && typeof c === 'object' && typeof (c as BrowserFindTextCandidate).tagName === 'string' + ) + if (candidates.length === 0) { + this.trace(tab, 'find_text', `未找到 ${text || selector}`) + return { + tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, + matched: 0, elements: [], candidates: [], ...(diag !== undefined ? { diag } : {}) + } + } + const picked = query.index ?? 1 + if (!Number.isInteger(picked) || picked < 1 || picked > candidates.length) { + throw new Error(`index 必须是 1 到 ${candidates.length} 之间的整数。`) + } + if (candidates.length > 1 && query.index === undefined) { + this.trace(tab, 'find_text', `「${text || selector}」命中 ${candidates.length} 处，等待指定 index`) + return { + tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, + matched: candidates.length, elements: [], candidates, ...(diag !== undefined ? { diag } : {}) + } + } + const candidate = candidates[picked - 1] + if (candidate.iframe === true) { + throw new Error('目标元素位于 iframe 内，无法直接注册 ref：请基于 browser_screenshot 用 browser_click_point 点击（候选列表 rect 已换算为页面顶层坐标）。') + } + // 选定阶段：给第 N 个命中打临时 data 属性，再经 DOM 域换成 backendNodeId。 + // （DOM.querySelector 不穿 shadow DOM，shadow 内的命中会在下一步报"未找到"， + // 报错信息会引导改用坐标点击） + const tagValue = Math.random().toString(36).slice(2, 10) + const pickPayload = await this.evaluateOnPage( + tab, + `((${FIND_TEXT_FINDER})(${JSON.stringify({ ...baseInput, pickIndex: picked, tagValue })}))` + ) as { ok?: boolean; tagged?: string } | null + if (!pickPayload || pickPayload.ok !== true) { + throw new Error('页面已变化，目标元素不存在了，请重新调用 browser_find_text。') + } + const documentResponse = await this.cdp(tab, 'DOM.getDocument', { depth: 0 }) + const rootId = (documentResponse.root as { nodeId?: number } | undefined)?.nodeId + if (typeof rootId !== 'number' || rootId <= 0) throw new Error('无法访问页面文档，请重试。') + const queryResponse = await this.cdp(tab, 'DOM.querySelector', { + nodeId: rootId, + selector: `[data-aso-find="${tagValue}"]` + }) + const taggedNodeId = (queryResponse as { nodeId?: number }).nodeId + if (typeof taggedNodeId !== 'number' || taggedNodeId === 0) { + throw new Error('目标元素无法转换为可点击引用（可能位于 shadow DOM 内），请改用 browser_screenshot + browser_click_point。') + } + const describeResponse = await this.cdp(tab, 'DOM.describeNode', { nodeId: taggedNodeId }) + const backendNodeId = (describeResponse.node as { backendNodeId?: number } | undefined)?.backendNodeId + await this.cdp(tab, 'DOM.removeAttribute', { nodeId: taggedNodeId, name: 'data-aso-find' }).catch(() => undefined) + if (typeof backendNodeId !== 'number' || backendNodeId <= 0) throw new Error('无法解析目标元素，请重新定位。') + const elements = this.buildRefs(tab, [ + { backendNodeId, role: candidate.tagName, name: candidate.text, editable: false } + ]) + this.trace(tab, 'find_text', `定位「${(candidate.text || candidate.tagName).slice(0, 40)}」→ ${elements[0]?.ref}（共 ${candidates.length} 处）`) + return { + tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, + matched: candidates.length, picked, + elements, + candidates: candidates.length > 1 ? candidates : undefined + } + }) + } + private async clickRef(tab: TabRecord, ref: string, generation: number): Promise { const target = this.resolveRef(tab, ref) const { x, y } = await this.centerForRef(tab, ref, generation) @@ -762,6 +1003,40 @@ export class BrowserController { }) } + /** 截图坐标兜底点击：坐标以最近一次 browser_screenshot 图像的像素为准（按截图比例换算 DIP）。 + * 仅供 observe/find/find_text 都无法定位时使用（canvas、封闭 shadow DOM、混淆页面）。 */ + async clickPoint(x: number, y: number, tabId?: string): Promise { + if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0) throw new Error('坐标必须是非负数。') + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + this.agentTabId = tab.tabId + return this.runAgentOp(tab, async () => { + const bounds = tab.view.getBounds() + if (bounds.width <= 0 || bounds.height <= 0) throw new Error('浏览器面板当前不可见，无法按坐标点击。') + const scale = tab.lastShotScale > 0 ? tab.lastShotScale : 1 + const dipX = x / scale + const dipY = y / scale + if (dipX >= bounds.width || dipY >= bounds.height) { + throw new Error( + `坐标越界：换算后 (${Math.round(dipX)}, ${Math.round(dipY)}) 超出页面 ${Math.round(bounds.width)}×${Math.round(bounds.height)}，请基于最新截图重新取坐标。` + ) + } + // 点击位置标记（宿主固定脚本，纯视觉反馈） + const marker = + '((p)=>{try{const d=document.createElement("div");d.style.cssText="position:fixed;left:"+(p.x-9)+"px;top:"+(p.y-9)+' + + '"px;width:18px;height:18px;border:2px solid #f59e0b;border-radius:50%;pointer-events:none;z-index:2147483647;box-sizing:border-box";' + + 'document.documentElement.appendChild(d);setTimeout(()=>d.remove(),700);return true}catch(e){return false}})(' + + JSON.stringify({ x: dipX, y: dipY }) + + ')' + await this.cdp(tab, 'Runtime.evaluate', { expression: marker, returnByValue: true }).catch(() => undefined) + await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: dipX, y: dipY }) + await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: dipX, y: dipY, button: 'left', clickCount: 1 }) + await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: dipX, y: dipY, button: 'left', clickCount: 1 }) + this.trace(tab, 'click_point', `点击截图坐标 (${Math.round(x)}, ${Math.round(y)})`) + return this.buildState() + }) + } + async fill(ref: string, text: string, tabId?: string): Promise { if (text.length > MAX_FILL_CHARS) throw new Error(`单次输入不能超过 ${MAX_FILL_CHARS} 个字符。`) this.assertRiskAcknowledged() @@ -930,6 +1205,9 @@ export class BrowserController { const tab = this.getAgentTab(tabId) return this.runAgentOp(tab, async () => { const image = await tab.view.webContents.capturePage() + // 记录 像素/DIP 比例，browser_click_point 换算截图坐标用（capturePage 按显示 DPR 放大） + const bounds = tab.view.getBounds() + if (bounds.width > 0) tab.lastShotScale = image.getSize().width / bounds.width this.trace(tab, 'screenshot', '截取当前页面') return { tabId: tab.tabId, url: tab.url, title: tab.title, dataUrl: image.toDataURL() } }) diff --git a/apps/desktop/src/main/browser/browser-policy.ts b/apps/desktop/src/main/browser/browser-policy.ts index 30f6a7e..7132e5d 100644 --- a/apps/desktop/src/main/browser/browser-policy.ts +++ b/apps/desktop/src/main/browser/browser-policy.ts @@ -78,6 +78,9 @@ export const INTERACTIVE_AX_ROLES = new Set([ /** 无名称也值得返回的 role（依赖 role 本身可定位） */ export const NAMELESS_ALLOWED_ROLES = new Set(['button', 'textfield', 'textbox', 'link', 'checkbox', 'combobox']) +/** browser_find_text 单次返回的候选上限（DOM 直查不受快照配额约束，巨页防爆量） */ +export const MAX_BROWSER_FIND_TEXT_RESULTS = 40 + export function resolveBrowserObserveMaxElements(requested?: number): number { if (typeof requested !== 'number' || !Number.isFinite(requested)) return DEFAULT_BROWSER_OBSERVE_MAX_ELEMENTS return Math.min(MAX_BROWSER_OBSERVE_MAX_ELEMENTS, Math.max(MIN_BROWSER_OBSERVE_MAX_ELEMENTS, Math.round(requested))) diff --git a/apps/desktop/src/main/tools/browser-tools.ts b/apps/desktop/src/main/tools/browser-tools.ts index 54e6fc7..4f672a2 100644 --- a/apps/desktop/src/main/tools/browser-tools.ts +++ b/apps/desktop/src/main/tools/browser-tools.ts @@ -8,7 +8,10 @@ import { getBrowserController } from '../browser/browser-controller' * 使用范式（与 Playwright MCP / Proma 一致的 AX 快照 + ref 体系）： * 1. browser_observe / browser_find 拿到带 ref 的可交互元素快照； * 2. 用 ref 调 click / fill / act 等操作；页面变化后旧 ref 失效，必须重新观察； - * 3. 语义结构足够时优先 observe，需要视觉验证时才 browser_screenshot。 + * 3. 语义结构足够时优先 observe，需要视觉验证时才 browser_screenshot； + * 4. 目标不在 AX 快照里（纯 div/span + JS 点击的元素，如商品规格/容量按钮）时， + * 用 browser_find_text 按可见文本或 CSS 选择器在 DOM 中定位，拿到 ref 后照常操作； + * 5. 一切定位手段都失效（canvas、封闭 shadow DOM）才 browser_screenshot + browser_click_point。 */ const TAB_ID_PARAM = Type.Optional( @@ -72,12 +75,34 @@ export const browserTools = [ } }), + defineTool({ + name: 'browser_find_text', + label: '按文本定位元素', + description: + '按可见文本或 CSS 选择器在页面 DOM 中定位元素并返回新 ref（text 与 selector 二选一）。不依赖无障碍树，专治 observe/find 快照里缺失的纯 div/span 可点击元素（商品规格、容量按钮、自定义控件）。text 匹配元素自身的直接文本（空白归一后精确匹配；contains=true 改为包含）。多个命中时不发 ref，返回候选列表（含 rect/上下文），带 index 重调以选定。命中并注册 ref 时作废旧 ref。支持开放 shadow DOM 与同源 iframe（iframe 内元素只能看候选，点击需改用坐标）。注意：快照配额会挤掉普通文本节点，即使 observe 看似完整，找不到目标时也应优先改用本工具。', + promptSnippet: 'browser_find_text: 按文本/选择器定位（AX 快照缺失时用）', + parameters: Type.Object({ + text: Type.Optional(Type.String({ description: '元素自身可见文本，如 "256GB"' })), + selector: Type.Optional(Type.String({ description: 'CSS 选择器（文本重复或目标无文本时用）' })), + contains: Type.Optional(Type.Boolean({ description: '文本包含匹配（默认精确匹配）' })), + index: Type.Optional(Type.Number({ minimum: 1, description: '多个候选时指定第几个（1 起，来自上次返回的候选列表）' })), + tabId: TAB_ID_PARAM + }), + async execute(_toolCallId, params) { + const result = await getBrowserController().findText( + { text: params.text, selector: params.selector, contains: params.contains, index: params.index }, + params.tabId + ) + return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result } + } + }), + defineTool({ name: 'browser_click', label: '点击元素', - description: '点击最新 observe/find 结果中的指定 ref。页面会短暂高亮目标。点击导致的重渲染会让旧 ref 失效，操作后需重新观察。', + description: '点击最新 observe/find/find_text 结果中的指定 ref。页面会短暂高亮目标。点击导致的重渲染会让旧 ref 失效，操作后需重新观察。', promptSnippet: 'browser_click: 点击 ref', - parameters: Type.Object({ ref: Type.String({ description: '元素引用（来自 observe/find）' }), tabId: TAB_ID_PARAM }), + parameters: Type.Object({ ref: Type.String({ description: '元素引用（来自 observe/find/find_text）' }), tabId: TAB_ID_PARAM }), async execute(_toolCallId, params) { const state = await getBrowserController().click(params.ref, params.tabId) return { content: [{ type: 'text', text: JSON.stringify({ clicked: params.ref, url: state.tabs.find((t) => t.tabId === state.activeTabId)?.url }) }], details: state } @@ -107,6 +132,23 @@ export const browserTools = [ } }), + defineTool({ + name: 'browser_click_point', + label: '坐标点击（兜底）', + description: + '点击最近一次 browser_screenshot 图像上的像素坐标 (x, y)。仅当 observe/find/find_text 都无法定位目标时的最后手段（canvas、封闭 shadow DOM、混淆页面）。点击前必须先截最新图；坐标越界会直接报错。', + promptSnippet: 'browser_click_point: 截图坐标兜底点击（最后手段）', + parameters: Type.Object({ + x: Type.Number({ minimum: 0, description: '截图图像上的 x 像素' }), + y: Type.Number({ minimum: 0, description: '截图图像上的 y 像素' }), + tabId: TAB_ID_PARAM + }), + async execute(_toolCallId, params) { + const state = await getBrowserController().clickPoint(params.x, params.y, params.tabId) + return { content: [{ type: 'text', text: JSON.stringify({ clicked: [params.x, params.y], url: state.tabs.find((t) => t.tabId === state.activeTabId)?.url }) }], details: state } + } + }), + defineTool({ name: 'browser_fill', label: '填写输入框',