import { getLogger } from '../logger' const log = getLogger('browser') import { app, BrowserWindow, session as electronSession, WebContentsView, type Rectangle, type Session } from 'electron' import { existsSync, readFileSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import type { BrowserImportInfo, BrowserTabInfo, BrowserUiState } from '@shared/types' import { BrowserCdpTimeoutError, BROWSER_OBSERVE_TIMEOUT_MS, resolveBrowserObserveAxDepth, withBrowserCdpTimeout, type CdpResponse } from './browser-cdp' import { assertSafeBrowserUrl, browserObservationNameLimit, MAX_BROWSER_FIND_TEXT_RESULTS, NAMELESS_ALLOWED_ROLES, prioritizeBrowserObservationCandidates, resolveBrowserDestination, resolveBrowserObserveMaxElements } from './browser-policy' /** * 内置受管浏览器控制器。 * * 架构(参考 Proma 的成熟实现,适配本应用单窗口/单聊天会话形态): * - 每个标签页 = 一个原生 WebContentsView,通过 contentView.addChildView 呈现; * 渲染层 BrowserSlot 量出占位矩形,经 IPC 同步边界(原生视图天然盖在 DOM 之上)。 * - 页面观察/操作全部走 CDP(webContents.debugger):AX tree 快照 + ref 代际机制 * + Input.dispatch* 真实输入事件;不在页面里执行模拟点击。 * - 登录态隔离在 persist:aso-browser 分区;本地浏览器配置导入由 browser-profile.ts * 直接迁移分区目录数据(Cookies/Local Storage + 密钥移植)。 */ const PARTITION = 'persist:aso-browser' // 新标签页默认地址:google.com 在目标市场(国内)不可达,新开标签会长时间白屏, // 误以为浏览器故障;用可稳定访问的导航页 const NEW_TAB_URL = 'https://www.baidu.com/' const MAX_TABS = 12 const MAX_FILL_CHARS = 10_000 const MAX_SCRIPT_CHARS = 20_000 const MAX_EXTRACT_CHARS = 50_000 interface RefEntry { backendNodeId: number generation: number label: string editable: boolean } interface TabRecord { tabId: string view: WebContentsView createdAt: number url: string title: string loading: boolean /** ref 表与代际号:每次观察/导航后旧 ref 失效 */ refs: Map generation: number /** 同一 tab 的 Agent 操作串行化,避免 CDP 命令交错 */ commandTail: Promise lastLayoutRevision: number attached: boolean /** 最近一次截图的 像素/DIP 比例(browser_click_point 换算截图坐标用) */ lastShotScale: number } interface BrowserSettings { riskAcknowledged: boolean imported?: BrowserImportInfo | null } export interface BrowserObservation { tabId: string url: string title: string generation: number elements: Array<{ ref: string; role: string; name: string; editable: boolean }> } /** browser_find_text 的候选描述。rect 为视口 CSS 像素(iframe 内元素已换算为顶层页面坐标) */ export interface BrowserFindTextCandidate { index: number tagName: string text: string rect: { x: number; y: number; w: number; h: number } context: string /** true = 位于 iframe 内(不能直接注册 ref 点击,需改用坐标点击) */ iframe?: boolean } export interface BrowserFindTextResult extends BrowserObservation { matched: number picked?: number candidates?: BrowserFindTextCandidate[] } export interface BrowserWaitCondition { kind: 'url' | 'text' | 'selector' value: string } function textValue(v: unknown): string { if (v && typeof v === 'object') { const value = (v as { value?: unknown }).value if (typeof value === 'string') return value } return '' } function isEditableAxNode(node: Record): boolean { const role = textValue(node.role).toLowerCase() if (role === 'textfield' || role === 'textbox' || role === 'searchbox') return true const props = Array.isArray(node.properties) ? (node.properties as Array>) : [] for (const p of props) { if (String(p.name) === 'editable') { const inner = p.value as { value?: unknown } | undefined if (inner && typeof inner.value === 'boolean') return inner.value } } return false } /** 导航键 → CDP key 事件参数(rawKeyDown + VK code 才会触发默认行为:滚动/提交/焦点移动) */ const KEY_EVENTS: Record = { enter: { key: 'Enter', code: 'Enter', vk: 13 }, tab: { key: 'Tab', code: 'Tab', vk: 9 }, escape: { key: 'Escape', code: 'Escape', vk: 27 }, backspace: { key: 'Backspace', code: 'Backspace', vk: 8 }, delete: { key: 'Delete', code: 'Delete', vk: 46 }, pageup: { key: 'PageUp', code: 'PageUp', vk: 33 }, pagedown: { key: 'PageDown', code: 'PageDown', vk: 34 }, home: { key: 'Home', code: 'Home', vk: 36 }, end: { key: 'End', code: 'End', vk: 35 }, arrowup: { key: 'ArrowUp', code: 'ArrowUp', vk: 38 }, arrowdown: { key: 'ArrowDown', code: 'ArrowDown', vk: 40 }, arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft', vk: 37 }, arrowright: { key: 'ArrowRight', code: 'ArrowRight', vk: 39 }, space: { key: ' ', code: 'Space', vk: 32 } } /** 页面内文本/选择器查找器(宿主固定函数,参数经 JSON 注入,非任意脚本)。 * 遍历主文档 + 开放 shadow DOM + 同源 iframe(深度受限),按元素自身直接可见文本或 * CSS 选择器匹配。返回纯可序列化数据(results 元数据 + diag 诊断 + pick 模式打标结果)。 * 背景:纯 div/span + JS 点击的元素(商品规格等)不在 AX 快照里,ref 体系够不到它们。 * 注意:结果回传只用 returnByValue 序列化——Runtime.callFunctionOn 跨命令持有 * RemoteObject 引用在 Electron debugger 下不可靠(this 绑定丢失,恒返回 undefined)。 */ const FIND_TEXT_FINDER = String.raw`function(input) { const norm = (s) => (s || '').replace(/\s+/g, ' ').trim() const results = [] const docs = [] let visitedTotal = 0 const collectDocs = (doc, depth) => { if (!doc || docs.indexOf(doc) >= 0 || depth > 3) return docs.push(doc) let frames = [] try { frames = Array.prototype.slice.call(doc.querySelectorAll('iframe')) } catch (err) {} for (const frame of frames) { try { collectDocs(frame.contentDocument, depth + 1) } catch (err) {} } } collectDocs(document, 0) if (input.selector) { try { document.querySelector(input.selector) } catch (err) { return { error: '选择器语法非法' } } } const ownText = (el) => { let text = '' for (let node = el.firstChild; node; node = node.nextSibling) { if (node.nodeType === 3) text += node.nodeValue } return norm(text) } const contextOf = (el) => { let parent = el.parentElement while (parent) { const text = norm(parent.textContent) if (text) return text.slice(0, 60) parent = parent.parentElement } return '' } const visible = (el) => { try { const rect = el.getBoundingClientRect() if (rect.width < 1 || rect.height < 1) return false const view = el.ownerDocument.defaultView if (!view) return false const style = view.getComputedStyle(el) return style.visibility !== 'hidden' && style.pointerEvents !== 'none' } catch (err) { return false } } const toTopCoords = (el) => { let doc = el.ownerDocument let x = 0 let y = 0 let guard = 0 while (doc && doc !== document && guard++ < 5) { const findHost = (d) => { let frames = [] try { frames = Array.prototype.slice.call(d.querySelectorAll('iframe')) } catch (err) {} for (const frame of frames) { if (frame.contentDocument === doc) return frame const nested = findHost(frame.contentDocument) if (nested) return nested } return null } const host = findHost(document) if (!host) break const hostRect = host.getBoundingClientRect() x += hostRect.x y += hostRect.y doc = host.ownerDocument } return { x, y } } const hits = [] const walk = (root) => { let iter = null try { iter = root.createTreeWalker(root, NodeFilter.SHOW_ELEMENT) } catch (err) { return } let visited = 0 let node while ((node = iter.nextNode())) { if (++visited > 20000) break visitedTotal++ const el = node if (el.shadowRoot) walk(el.shadowRoot) let hit = false let label = '' if (input.selector) { try { hit = el.matches(input.selector) } catch (err) { hit = false } if (hit) { label = ownText(el) if (!label) { try { label = norm(el.getAttribute('aria-label') || el.getAttribute('title') || '') } catch (err) {} } } } else { label = ownText(el) if (label) hit = input.contains ? label.indexOf(input.text) >= 0 : label === input.text } if (hit && visible(el)) hits.push({ el, label }) } } for (const doc of docs) { if (doc.documentElement) walk(doc) } if (input.pickIndex !== undefined && input.pickIndex !== null) { const hit = hits[input.pickIndex - 1] if (!hit) return { ok: false, diag: { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href) } } hit.el.setAttribute('data-aso-find', input.tagValue) const rect = hit.el.getBoundingClientRect() const offset = hit.el.ownerDocument === document ? { x: 0, y: 0 } : toTopCoords(hit.el) return { ok: true, tagged: hit.label.slice(0, 80), rect: { x: Math.round(rect.x + offset.x), y: Math.round(rect.y + offset.y) }, diag: { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href) } } } for (const hit of hits.slice(0, input.limit)) { const rect = hit.el.getBoundingClientRect() const offset = hit.el.ownerDocument === document ? { x: 0, y: 0 } : toTopCoords(hit.el) results.push({ index: results.length + 1, tagName: hit.el.tagName.toLowerCase(), text: hit.label.slice(0, 80), rect: { x: Math.round(rect.x + offset.x), y: Math.round(rect.y + offset.y), w: Math.round(rect.width), h: Math.round(rect.height) }, context: contextOf(hit.el), iframe: hit.el.ownerDocument !== document }) } const diag = { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href), readyState: document.readyState, nodeFilter: typeof NodeFilter } return { results, diag } }` export class BrowserController { private owner: BrowserWindow | null = null private browserSession: Session private tabs = new Map() private activeTabId: string | null = null private agentTabId: string | null = null private listeners = new Set<(state: BrowserUiState) => void>() private traceText: string | null = null private settings: BrowserSettings = { riskAcknowledged: false } /** 最近关闭的页签(新的在前,供「最近关闭的标签页」恢复) */ private closedTabs: Array<{ title: string; url: string; closedAt: number }> = [] constructor() { this.browserSession = electronSession.fromPartition(PARTITION) // 诚实的 UA:保留 Chromium token 保证站点兼容,去掉 Electron 标识 this.browserSession.setUserAgent( this.browserSession.getUserAgent().replace(/Electron\/[\d.]+/, `InkreachAgent/${app.getVersion()}`) ) // 网页权限请求(摄像头/定位等)默认拒绝 this.browserSession.setPermissionRequestHandler((_wc, _permission, callback) => callback(false)) // 下载留在受管浏览器内,落到系统下载目录(重名自动加序号) this.browserSession.on('will-download', (_event, item) => { const dir = app.getPath('downloads') const base = item.getFilename() || 'download' const dot = base.lastIndexOf('.') const stem = dot > 0 ? base.slice(0, dot) : base const ext = dot > 0 ? base.slice(dot) : '' let target = join(dir, base) for (let i = 1; existsSync(target) && i < 100; i++) { target = join(dir, `${stem} (${i})${ext}`) } item.setSavePath(target) }) this.settings = this.loadSettings() } // ---- 设置持久化 ---- private get settingsFile(): string { return join(app.getPath('userData'), 'aso-browser.json') } private loadSettings(): BrowserSettings { try { const raw = JSON.parse(readFileSync(this.settingsFile, 'utf8')) as Partial return { riskAcknowledged: raw.riskAcknowledged === true, imported: raw.imported ?? null } } catch { return { riskAcknowledged: false } } } private saveSettings(): void { try { writeFileSync(this.settingsFile, JSON.stringify(this.settings, null, 2), 'utf8') } catch (err) { log.error('save settings failed:', { err: err }) } } /** 配置导入完成后由 browser-profile.ts 回写摘要 */ setImportedInfo(info: BrowserImportInfo | null): void { this.settings.imported = info this.saveSettings() this.emitState() } get importedInfo(): BrowserImportInfo | null { return this.settings.imported ?? null } /** 当前是否有活跃标签页(导入配置前必须没有) */ get hasTabs(): boolean { return this.tabs.size > 0 } // ---- 窗口与状态广播 ---- setOwnerWindow(win: BrowserWindow): void { this.owner = win } handleWindowClosed(): void { for (const tab of this.tabs.values()) { try { if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach() } catch { // 已销毁 } try { tab.view.setVisible(false) } catch { // 已销毁 } } this.tabs.clear() this.activeTabId = null this.agentTabId = null this.owner = null } onState(fn: (state: BrowserUiState) => void): () => void { this.listeners.add(fn) return () => this.listeners.delete(fn) } private buildState(): BrowserUiState { const tabs: BrowserTabInfo[] = [...this.tabs.values()] .sort((a, b) => a.createdAt - b.createdAt) .map((t) => ({ tabId: t.tabId, url: t.url, title: t.title, loading: t.loading })) return { open: tabs.length > 0, riskAcknowledged: this.settings.riskAcknowledged, tabs, activeTabId: this.activeTabId, recentClosed: this.closedTabs.map(({ title, url }) => ({ title, url })), trace: this.traceText, imported: this.settings.imported ?? null } } getState(): BrowserUiState { return this.buildState() } private emitState(): void { const state = this.buildState() for (const fn of this.listeners) { try { fn(state) } catch (err) { log.error('state listener error:', { err: err }) } } } private trace(tab: TabRecord, action: string, detail: string): void { this.traceText = `[${action}] ${detail}` void tab this.emitState() } // ---- 风险告知门禁 ---- acknowledgeRisk(): void { this.settings.riskAcknowledged = true this.saveSettings() this.emitState() } private assertRiskAcknowledged(): void { if (!this.settings.riskAcknowledged) { throw new Error('内置浏览器首次使用尚未确认风险告知:请在应用右侧浏览器面板中阅读并点击「我已了解,开始使用」后重试。') } } // ---- 会话与标签页 ---- private ensureBrowser(): TabRecord { if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪,无法使用内置浏览器。') if (this.tabs.size === 0) this.createTab(NEW_TAB_URL) const active = this.activeTabId ? this.tabs.get(this.activeTabId) : null const fallback = this.tabs.values().next().value const tab = active ?? fallback if (!tab) throw new Error('浏览器标签页不可用。') this.emitState() return tab } private invalidateTabDocument(tab: TabRecord): void { tab.generation += 1 tab.refs.clear() } private createTab(initialUrl: string, viewOptions?: Electron.WebContentsViewConstructorOptions): TabRecord { if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪,无法创建浏览器标签。') if (this.tabs.size >= MAX_TABS) throw new Error(`浏览器标签页已达上限(${MAX_TABS}),请先关闭不需要的标签。`) const tabId = `tab-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` const view = new WebContentsView( viewOptions ?? { webPreferences: { partition: PARTITION, nodeIntegration: false, contextIsolation: true, sandbox: true, webSecurity: true } } ) const tab: TabRecord = { tabId, view, createdAt: Date.now(), url: '', title: '新标签页', loading: false, refs: new Map(), generation: 0, commandTail: Promise.resolve(), lastLayoutRevision: 0, attached: false, lastShotScale: 1 } view.setVisible(false) // popup 收编为受管同级标签(必须同步返回 webContents,故提前建好 TabRecord) view.webContents.setWindowOpenHandler(({ url }) => { try { assertSafeBrowserUrl(url) } catch { this.trace(tab, 'popup', `已阻止不安全的新窗口: ${url.slice(0, 80)}`) return { action: 'deny' } } if (this.tabs.get(tabId) !== tab || this.tabs.size >= MAX_TABS) return { action: 'deny' } return { action: 'allow', outlivesOpener: true, createWindow: (options) => { const popup = this.createTab(url, options) this.selectTab(popup.tabId) this.trace(popup, 'popup', '网页打开了新标签页') return popup.view.webContents } } }) const updateNav = (): void => { const wc = view.webContents if (wc.isDestroyed()) return try { tab.url = wc.getURL() tab.title = wc.getTitle() || tab.url tab.loading = wc.isLoading() } catch { // 竞态时忽略 } this.emitState() } view.webContents.on('did-start-loading', () => { tab.loading = true this.invalidateTabDocument(tab) this.emitState() }) view.webContents.on('did-stop-loading', updateNav) view.webContents.on('page-title-updated', updateNav) view.webContents.on('did-navigate', () => { this.invalidateTabDocument(tab) updateNav() }) view.webContents.on('did-navigate-in-page', () => { this.invalidateTabDocument(tab) updateNav() }) view.webContents.on('destroyed', () => { this.tabs.delete(tabId) this.removeView(view) if (this.activeTabId === tabId) this.activeTabId = this.tabs.keys().next().value ?? null if (this.agentTabId === tabId) this.agentTabId = this.activeTabId this.emitState() }) try { view.webContents.debugger.attach('1.3') } catch (err) { log.warn('CDP attach 失败:', { err: err }) } this.tabs.set(tabId, tab) if (!this.activeTabId) this.activeTabId = tabId // popup 场景(带 viewOptions)由 Electron 的 createWindow 机制自行导航,避免双重加载 if (initialUrl && !viewOptions) void view.webContents.loadURL(assertSafeBrowserUrl(initialUrl)).catch(() => undefined) return tab } private removeView(view: WebContentsView): void { try { this.owner?.contentView.removeChildView(view) } catch { // 窗口已销毁 } try { view.webContents.close() } catch { // 已销毁 } } listTabs(): BrowserUiState { return this.buildState() } newTab(url?: string, byAgent = true): BrowserUiState { this.assertRiskAcknowledged() const tab = this.createTab(url ? resolveBrowserDestination(url) : NEW_TAB_URL) this.activeTabId = tab.tabId if (byAgent) this.agentTabId = tab.tabId this.emitState() return this.buildState() } selectTab(tabId: string): BrowserUiState { const tab = this.tabs.get(tabId) if (!tab) throw new Error(`标签页不存在: ${tabId}`) this.activeTabId = tabId try { tab.view.webContents.focus() } catch { // 页面可能正在加载 } this.emitState() return this.buildState() } closeTab(tabId?: string): BrowserUiState { const id = tabId ?? this.activeTabId if (!id) return this.buildState() const tab = this.tabs.get(id) if (!tab) return this.buildState() try { if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach() } catch { // 已销毁 } this.removeView(tab.view) this.tabs.delete(id) // 记入「最近关闭的标签页」(新的在前,最多 10 条) this.closedTabs.unshift({ title: tab.title || tab.url || '新标签页', url: tab.url, closedAt: Date.now() }) this.closedTabs = this.closedTabs.slice(0, 10) if (this.activeTabId === id) this.activeTabId = this.tabs.keys().next().value ?? null if (this.agentTabId === id) this.agentTabId = this.activeTabId this.emitState() return this.buildState() } /** 关闭整个受管浏览器(销毁全部标签;导入配置前也会走这里) */ close(): BrowserUiState { for (const tabId of [...this.tabs.keys()]) this.closeTab(tabId) this.agentTabId = null this.traceText = null return this.buildState() } // ---- 渲染层布局同步 ---- applyLayout(layout: { tabId: string; visible: boolean; bounds: Rectangle; revision: number }): void { const tab = this.tabs.get(layout.tabId) if (!tab || !this.owner || this.owner.isDestroyed()) return if (layout.revision <= tab.lastLayoutRevision) return tab.lastLayoutRevision = layout.revision try { if (layout.visible && layout.bounds.width > 4 && layout.bounds.height > 4) { if (!tab.attached) { this.owner.contentView.addChildView(tab.view) tab.attached = true } tab.view.setBounds(layout.bounds) tab.view.setVisible(true) } else { tab.view.setVisible(false) } } catch (err) { log.error('apply layout failed:', { err: err }) } } // ---- 用户导航(地址栏/按钮) ---- navigateDisplay(input: string): BrowserUiState { this.assertRiskAcknowledged() const tab = this.ensureBrowser() void tab.view.webContents.loadURL(resolveBrowserDestination(input)).catch(() => undefined) this.trace(tab, 'navigate', `打开 ${input.slice(0, 80)}`) return this.buildState() } goBack(): BrowserUiState { this.assertRiskAcknowledged() const tab = this.ensureBrowser() if (tab.view.webContents.navigationHistory.canGoBack()) tab.view.webContents.navigationHistory.goBack() return this.buildState() } goForward(): BrowserUiState { this.assertRiskAcknowledged() const tab = this.ensureBrowser() if (tab.view.webContents.navigationHistory.canGoForward()) tab.view.webContents.navigationHistory.goForward() return this.buildState() } reload(): BrowserUiState { this.assertRiskAcknowledged() const tab = this.ensureBrowser() tab.view.webContents.reload() return this.buildState() } // ---- CDP 基础 ---- private cdp(tab: TabRecord, method: string, params?: Record, timeoutMs?: number): Promise { const debuggerClient = tab.view.webContents.debugger if (!debuggerClient.isAttached()) throw new Error('浏览器调试通道不可用,请重新打开该标签页。') return withBrowserCdpTimeout(() => debuggerClient.sendCommand(method, params) as Promise, method, timeoutMs).catch( (error: unknown) => { if (error instanceof BrowserCdpTimeoutError) this.recoverDebugger(tab, method) throw error } ) } /** 超时后重连 debugger,避免一个卡死页面让后续浏览器调用全部失败 */ private recoverDebugger(tab: TabRecord, method: string): void { try { const debuggerClient = tab.view.webContents.debugger if (debuggerClient.isAttached()) debuggerClient.detach() debuggerClient.attach('1.3') log.warn(`CDP ${method} timeout, reattached`) } catch { // 页面已关闭 } } /** Agent 的 tab 内操作串行化(用户操作不排队) */ private runAgentOp(tab: TabRecord, fn: () => Promise): Promise { const next = tab.commandTail.then(fn, fn) tab.commandTail = next.catch(() => undefined) return next } private getAgentTab(tabId?: string): TabRecord { if (tabId) { const explicit = this.tabs.get(tabId) if (!explicit) throw new Error('目标标签页不存在,请先 browser_list_tabs 并用 browser_new_tab/browser_select_tab 选择。') return explicit } const id = this.agentTabId ?? this.activeTabId const tab = id ? this.tabs.get(id) : null if (tab) return tab // Agent 首次操作时还没有任何标签页:自动开一个工作页 return this.ensureBrowser() } private assertCurrentDocument(tab: TabRecord, generation: number): void { if (tab.generation !== generation) { throw new Error('页面已变化(导航或重渲染),旧元素引用失效:请重新 browser_observe 后再操作。') } } private resolveRef(tab: TabRecord, ref: string): RefEntry { const entry = tab.refs.get(ref) if (!entry || entry.generation !== tab.generation) { throw new Error('元素引用已失效,请先重新调用 browser_observe。') } return entry } /** AX ref → 视口中心坐标(先滚动到可见再读 box;懒加载列表滚动后坐标会变,不能复用旧值) */ private async centerForRef(tab: TabRecord, ref: string, generation: number): Promise<{ x: number; y: number }> { this.assertCurrentDocument(tab, generation) const { backendNodeId } = this.resolveRef(tab, ref) await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId }) this.assertCurrentDocument(tab, generation) const box = await this.cdp(tab, 'DOM.getBoxModel', { backendNodeId }) this.assertCurrentDocument(tab, generation) const model = box.model as Record | undefined const quad = Array.isArray(model?.content) ? (model?.content as unknown[]) : [] if (quad.length < 8 || !quad.every((v) => typeof v === 'number')) { throw new Error('目标元素当前不可点击,请重新观察页面。') } const n = quad as number[] return { x: (n[0] + n[2] + n[4] + n[6]) / 4, y: (n[1] + n[3] + n[5] + n[7]) / 4 } } /** 点击前在页面上短暂高亮目标,用户能看清 Agent 正在操作什么 */ private async highlightAgentTarget(tab: TabRecord, backendNodeId: number): Promise { try { const resolved = await this.cdp(tab, 'DOM.resolveNode', { backendNodeId }) const objectId = (resolved.object as { objectId?: string } | undefined)?.objectId if (!objectId) return await this.cdp(tab, 'Runtime.callFunctionOn', { functionDeclaration: 'function(){const r=this.getBoundingClientRect();const d=document.createElement("div");' + 'd.style.cssText=`position:fixed;left:${r.left-2}px;top:${r.top-2}px;width:${r.width+4}px;height:${r.height+4}px;`+' + '"outline:2px solid #f59e0b;outline-offset:1px;pointer-events:none;z-index:2147483647;";' + 'document.documentElement.appendChild(d);setTimeout(()=>d.remove(),700);return true}', objectId, returnByValue: true }) } catch { // 高亮失败不影响操作 } } // ---- Agent 工具实现 ---- async navigate(url: string, tabId?: string): Promise { this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) this.agentTabId = tab.tabId this.activeTabId = tab.tabId const destination = resolveBrowserDestination(url) await tab.view.webContents.loadURL(destination) this.trace(tab, 'navigate', `打开 ${destination.slice(0, 80)}`) return this.buildState() } private parseAxNodes(response: CdpResponse): Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> { const nodes = Array.isArray(response.nodes) ? (response.nodes as Array>) : [] const candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> = [] for (const node of nodes) { if (!node) continue const backendNodeId = typeof node.backendDOMNodeId === 'number' ? node.backendDOMNodeId : 0 const role = textValue(node.role).toLowerCase() const name = textValue(node.name) const editable = isEditableAxNode(node) if (!backendNodeId || !role) continue if (!name && !editable && !NAMELESS_ALLOWED_ROLES.has(role)) continue candidates.push({ backendNodeId, role, name: name.slice(0, browserObservationNameLimit(role)), editable }) } return candidates } private buildRefs(tab: TabRecord, candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }>): BrowserObservation['elements'] { tab.generation += 1 tab.refs.clear() const elements: BrowserObservation['elements'] = [] for (const candidate of candidates) { const ref = `r${tab.generation}-${elements.length + 1}` tab.refs.set(ref, { backendNodeId: candidate.backendNodeId, generation: tab.generation, label: candidate.name ? `${candidate.role}「${candidate.name.slice(0, 80)}」` : candidate.role, editable: candidate.editable }) elements.push({ ref, role: candidate.role, name: candidate.name, editable: candidate.editable }) } return elements } async observe(tabId?: string, requestedMaxElements?: number): Promise { this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) this.agentTabId = tab.tabId return this.runAgentOp(tab, async () => { const maxElements = resolveBrowserObserveMaxElements(requestedMaxElements) const depth = resolveBrowserObserveAxDepth(maxElements) const response = await this.cdp( tab, 'Accessibility.getFullAXTree', { depth }, BROWSER_OBSERVE_TIMEOUT_MS ) const selected = prioritizeBrowserObservationCandidates(this.parseAxNodes(response), maxElements) const elements = this.buildRefs(tab, selected) this.trace(tab, 'observe', `读取到 ${elements.length}/${maxElements} 个元素(AX 深度 ${depth})`) return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements } }) } async find( query: { role?: string; name?: string; exact?: boolean; maxResults?: number }, tabId?: string ): Promise { const role = query.role?.trim().toLowerCase() const name = query.name?.trim().toLowerCase() const maxResults = query.maxResults ?? 20 if (!role && !name) throw new Error('语义定位至少需要 role 或 name。') if (!Number.isInteger(maxResults) || maxResults < 1 || maxResults > 50) throw new Error('maxResults 必须是 1 到 50 的整数。') this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) this.agentTabId = tab.tabId return this.runAgentOp(tab, async () => { // 过滤后再套观察配额:Find 能找到被紧凑 Observe 截掉的目标 const response = await this.cdp( tab, 'Accessibility.getFullAXTree', { depth: resolveBrowserObserveAxDepth(MAX_BROWSER_OBSERVE_DEPTH_FALLBACK) }, BROWSER_OBSERVE_TIMEOUT_MS ) const candidates = this.parseAxNodes(response).filter((candidate) => { const roleMatches = !role || candidate.role === role const candidateName = candidate.name.toLowerCase() const nameMatches = !name || (query.exact ? candidateName === name : candidateName.includes(name)) return roleMatches && nameMatches }) const elements = this.buildRefs(tab, candidates.slice(0, maxResults)) this.trace(tab, 'find', `语义定位到 ${elements.length} 个元素(${role ?? name})`) return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements } }) } /** 按可见文本/CSS 选择器在 DOM 中直查(不依赖无障碍树):命中后注册为普通 ref, * browser_click 等后续操作照常使用。多命中且未指定 index 时只返回候选列表不发 ref; * 仅在真正注册 ref 时才作废旧 ref(与 observe/find 的失效语义一致)。 */ async findText( query: { text?: string; selector?: string; contains?: boolean; index?: number }, tabId?: string ): Promise { const text = query.text?.trim().slice(0, 200) ?? '' const selector = query.selector?.trim().slice(0, 500) ?? '' if ((text ? 1 : 0) + (selector ? 1 : 0) !== 1) throw new Error('text 与 selector 必须且只能提供一个。') this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) this.agentTabId = tab.tabId return this.runAgentOp(tab, async () => { const baseInput = { text: text || undefined, selector: selector || undefined, contains: query.contains === true, limit: MAX_BROWSER_FIND_TEXT_RESULTS } const payload = await this.evaluateOnPage(tab, `((${FIND_TEXT_FINDER})(${JSON.stringify(baseInput)}))`) as | { error?: string; results?: unknown[]; diag?: unknown } | null if (!payload || typeof payload !== 'object') throw new Error('页面定位结果异常,请重试。') if (payload.error) throw new Error(`定位失败:${payload.error}`) const diag = payload.diag const candidates = (Array.isArray(payload.results) ? payload.results : []).filter( (c): c is BrowserFindTextCandidate => !!c && typeof c === 'object' && typeof (c as BrowserFindTextCandidate).tagName === 'string' ) if (candidates.length === 0) { this.trace(tab, 'find_text', `未找到 ${text || selector}`) return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, matched: 0, elements: [], candidates: [], ...(diag !== undefined ? { diag } : {}) } } const picked = query.index ?? 1 if (!Number.isInteger(picked) || picked < 1 || picked > candidates.length) { throw new Error(`index 必须是 1 到 ${candidates.length} 之间的整数。`) } if (candidates.length > 1 && query.index === undefined) { this.trace(tab, 'find_text', `「${text || selector}」命中 ${candidates.length} 处,等待指定 index`) return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, matched: candidates.length, elements: [], candidates, ...(diag !== undefined ? { diag } : {}) } } const candidate = candidates[picked - 1] if (candidate.iframe === true) { throw new Error('目标元素位于 iframe 内,无法直接注册 ref:请基于 browser_screenshot 用 browser_click_point 点击(候选列表 rect 已换算为页面顶层坐标)。') } // 选定阶段:给第 N 个命中打临时 data 属性,再经 DOM 域换成 backendNodeId。 // (DOM.querySelector 不穿 shadow DOM,shadow 内的命中会在下一步报"未找到", // 报错信息会引导改用坐标点击) const tagValue = Math.random().toString(36).slice(2, 10) const pickPayload = await this.evaluateOnPage( tab, `((${FIND_TEXT_FINDER})(${JSON.stringify({ ...baseInput, pickIndex: picked, tagValue })}))` ) as { ok?: boolean; tagged?: string } | null if (!pickPayload || pickPayload.ok !== true) { throw new Error('页面已变化,目标元素不存在了,请重新调用 browser_find_text。') } const documentResponse = await this.cdp(tab, 'DOM.getDocument', { depth: 0 }) const rootId = (documentResponse.root as { nodeId?: number } | undefined)?.nodeId if (typeof rootId !== 'number' || rootId <= 0) throw new Error('无法访问页面文档,请重试。') const queryResponse = await this.cdp(tab, 'DOM.querySelector', { nodeId: rootId, selector: `[data-aso-find="${tagValue}"]` }) const taggedNodeId = (queryResponse as { nodeId?: number }).nodeId if (typeof taggedNodeId !== 'number' || taggedNodeId === 0) { throw new Error('目标元素无法转换为可点击引用(可能位于 shadow DOM 内),请改用 browser_screenshot + browser_click_point。') } const describeResponse = await this.cdp(tab, 'DOM.describeNode', { nodeId: taggedNodeId }) const backendNodeId = (describeResponse.node as { backendNodeId?: number } | undefined)?.backendNodeId await this.cdp(tab, 'DOM.removeAttribute', { nodeId: taggedNodeId, name: 'data-aso-find' }).catch(() => undefined) if (typeof backendNodeId !== 'number' || backendNodeId <= 0) throw new Error('无法解析目标元素,请重新定位。') const elements = this.buildRefs(tab, [ { backendNodeId, role: candidate.tagName, name: candidate.text, editable: false } ]) this.trace(tab, 'find_text', `定位「${(candidate.text || candidate.tagName).slice(0, 40)}」→ ${elements[0]?.ref}(共 ${candidates.length} 处)`) return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, matched: candidates.length, picked, elements, candidates: candidates.length > 1 ? candidates : undefined } }) } private async clickRef(tab: TabRecord, ref: string, generation: number): Promise { const target = this.resolveRef(tab, ref) const { x, y } = await this.centerForRef(tab, ref, generation) this.assertCurrentDocument(tab, generation) await this.highlightAgentTarget(tab, target.backendNodeId) await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 }) await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 }) return target } async click(ref: string, tabId?: string): Promise { this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) return this.runAgentOp(tab, async () => { const generation = tab.generation const target = await this.clickRef(tab, ref, generation) this.trace(tab, 'click', `点击 ${target.label}`) return this.buildState() }) } /** 点击 + 等待合并为一次串行操作(已知点击后预期状态时优先使用) */ async act(ref: string, waitFor: BrowserWaitCondition | undefined, timeoutMs = 10_000, tabId?: string): Promise<{ state: BrowserUiState; matched: boolean }> { this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) return this.runAgentOp(tab, async () => { const generation = tab.generation const target = await this.clickRef(tab, ref, generation) this.trace(tab, 'click', `点击 ${target.label}`) if (!waitFor) return { state: this.buildState(), matched: true } const matched = await this.waitForInternal(tab, waitFor, timeoutMs) return { state: this.buildState(), matched } }) } async hover(ref: string, tabId?: string): Promise { this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) return this.runAgentOp(tab, async () => { const generation = tab.generation const target = this.resolveRef(tab, ref) const { x, y } = await this.centerForRef(tab, ref, generation) await this.highlightAgentTarget(tab, target.backendNodeId) this.assertCurrentDocument(tab, generation) await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y }) this.trace(tab, 'hover', `悬停 ${target.label}`) return this.buildState() }) } /** 截图坐标兜底点击:坐标以最近一次 browser_screenshot 图像的像素为准(按截图比例换算 DIP)。 * 仅供 observe/find/find_text 都无法定位时使用(canvas、封闭 shadow DOM、混淆页面)。 */ async clickPoint(x: number, y: number, tabId?: string): Promise { if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0) throw new Error('坐标必须是非负数。') this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) this.agentTabId = tab.tabId return this.runAgentOp(tab, async () => { const bounds = tab.view.getBounds() if (bounds.width <= 0 || bounds.height <= 0) throw new Error('浏览器面板当前不可见,无法按坐标点击。') const scale = tab.lastShotScale > 0 ? tab.lastShotScale : 1 const dipX = x / scale const dipY = y / scale if (dipX >= bounds.width || dipY >= bounds.height) { throw new Error( `坐标越界:换算后 (${Math.round(dipX)}, ${Math.round(dipY)}) 超出页面 ${Math.round(bounds.width)}×${Math.round(bounds.height)},请基于最新截图重新取坐标。` ) } // 点击位置标记(宿主固定脚本,纯视觉反馈) const marker = '((p)=>{try{const d=document.createElement("div");d.style.cssText="position:fixed;left:"+(p.x-9)+"px;top:"+(p.y-9)+' + '"px;width:18px;height:18px;border:2px solid #f59e0b;border-radius:50%;pointer-events:none;z-index:2147483647;box-sizing:border-box";' + 'document.documentElement.appendChild(d);setTimeout(()=>d.remove(),700);return true}catch(e){return false}})(' + JSON.stringify({ x: dipX, y: dipY }) + ')' await this.cdp(tab, 'Runtime.evaluate', { expression: marker, returnByValue: true }).catch(() => undefined) await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: dipX, y: dipY }) await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: dipX, y: dipY, button: 'left', clickCount: 1 }) await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: dipX, y: dipY, button: 'left', clickCount: 1 }) this.trace(tab, 'click_point', `点击截图坐标 (${Math.round(x)}, ${Math.round(y)})`) return this.buildState() }) } async fill(ref: string, text: string, tabId?: string): Promise { if (text.length > MAX_FILL_CHARS) throw new Error(`单次输入不能超过 ${MAX_FILL_CHARS} 个字符。`) this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) return this.runAgentOp(tab, async () => { const generation = tab.generation const target = this.resolveRef(tab, ref) if (!target.editable) throw new Error('目标元素不是可编辑字段,请重新观察后选择 input、textarea 或 contenteditable。') await this.highlightAgentTarget(tab, target.backendNodeId) await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backendNodeId }) this.assertCurrentDocument(tab, generation) await this.cdp(tab, 'DOM.focus', { backendNodeId: target.backendNodeId }) this.assertCurrentDocument(tab, generation) // 全选(Ctrl/Cmd+A)后 insertText 整段替换,不逐字符模拟 const selectAllModifier = process.platform === 'darwin' ? 4 : 2 await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'keyDown', key: 'a', code: 'KeyA', windowsVirtualKeyCode: 65, modifiers: selectAllModifier }) await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'keyUp', key: 'a', code: 'KeyA', windowsVirtualKeyCode: 65, modifiers: selectAllModifier }) this.assertCurrentDocument(tab, generation) await this.cdp(tab, 'Input.insertText', { text }) this.trace(tab, 'fill', `在 ${target.label} 输入 ${Array.from(text).length} 个字符`) return this.buildState() }) } async press(key: string, tabId?: string): Promise { this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) return this.runAgentOp(tab, async () => { const normalized = key.trim().toLowerCase().replace(/\s+/g, '') const keyEvent = KEY_EVENTS[normalized] if (keyEvent) { const event = { key: keyEvent.key, code: keyEvent.code, windowsVirtualKeyCode: keyEvent.vk } await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'rawKeyDown', ...event }) await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'keyUp', ...event }) this.trace(tab, 'press', `按下 ${keyEvent.key}`) } else { await this.cdp(tab, 'Input.insertText', { text: key }) this.trace(tab, 'press', `输入 ${Array.from(key).length} 个字符`) } return this.buildState() }) } async scroll( input: { deltaY?: number; position?: 'top' | 'bottom'; selector?: string }, tabId?: string ): Promise<{ before: { top: number; height: number }; after: { top: number; height: number }; moved: boolean }> { this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) return this.runAgentOp(tab, async () => { const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : '' const position = input.position === 'top' || input.position === 'bottom' ? input.position : undefined const deltaY = Math.max(-50_000, Math.min(50_000, Math.round(input.deltaY ?? 600))) // 固定表达式:数据由 JSON 注入,不拼接用户脚本 const expression = '((input)=>{const el=input.selector?document.querySelector(input.selector):(document.scrollingElement||document.documentElement);' + 'if(!el)return {ok:false};const read=()=>({top:Math.round(el.scrollTop),height:Math.round(el.scrollHeight)});' + 'const before=read();const top=input.position==="top"?0:input.position==="bottom"?Math.max(0,el.scrollHeight-el.clientHeight):el.scrollTop+input.deltaY;' + 'el.scrollTo({top});const after=read();return {ok:true,before,after}})(' + JSON.stringify({ selector, position, deltaY }) + ')' const response = await this.evaluateOnPage(tab, expression) const result = (response ?? {}) as { ok?: boolean; before?: { top: number; height: number }; after?: { top: number; height: number } } if (!result.ok || !result.before || !result.after) throw new Error('滚动目标不存在或不支持滚动,请检查 selector。') this.trace(tab, 'scroll', `滚动页面${selector ? `(${selector.slice(0, 40)})` : ''}`) return { before: result.before, after: result.after, moved: result.after.top !== result.before.top } }) } async extract( input: { selector?: string; maxChars?: number }, tabId?: string ): Promise<{ url: string; title: string; text: string; truncated: boolean }> { this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) return this.runAgentOp(tab, async () => { const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : '' const maxChars = Math.min(MAX_EXTRACT_CHARS, Math.max(500, Math.round(input.maxChars ?? MAX_EXTRACT_CHARS))) const expression = '((input)=>{const root=input.selector?document.querySelector(input.selector):document.body;' + 'if(!root)return null;const text=(root.innerText||"").replace(/\\n{3,}/g,"\\n\\n");' + 'return {text:text.slice(0,input.maxChars),truncated:text.length>input.maxChars}})(' + JSON.stringify({ selector, maxChars }) + ')' const response = await this.evaluateOnPage(tab, expression) const result = (response ?? {}) as { text?: string; truncated?: boolean } | null if (!result || typeof result.text !== 'string') throw new Error('提取目标不存在,请检查 selector。') this.trace(tab, 'extract', `提取正文 ${result.text.length} 字符${selector ? `(${selector.slice(0, 40)})` : ''}`) return { url: tab.url, title: tab.title, text: result.text, truncated: result.truncated === true } }) } async waitFor(condition: BrowserWaitCondition, timeoutMs = 10_000, tabId?: string): Promise<{ matched: boolean; url: string; title: string }> { if (!condition.value.trim()) throw new Error('等待条件不能为空。') if (!Number.isFinite(timeoutMs) || timeoutMs < 250 || timeoutMs > 30_000) throw new Error('等待超时必须在 250 到 30000 毫秒之间。') this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) return this.runAgentOp(tab, async () => { const matched = await this.waitForInternal(tab, condition, timeoutMs) this.trace(tab, 'wait', `${matched ? '已满足' : '超时未满足'}${condition.kind} 条件(${condition.value.slice(0, 40)})`) return { matched, url: tab.url, title: tab.title } }) } /** 轮询固定表达式(只读检查,不执行任意 JS) */ private async waitForInternal(tab: TabRecord, condition: BrowserWaitCondition, timeoutMs: number): Promise { const startedAt = Date.now() const payload = JSON.stringify(condition).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029') const expression = `(() => { const condition = ${payload}; try { ` + "if (condition.kind === 'url') return location.href.includes(condition.value); " + "if (condition.kind === 'text') return (document.body?.innerText || '').includes(condition.value); " + 'return !!document.querySelector(condition.value); } catch { return false; } })()' while (Date.now() - startedAt <= timeoutMs) { const result = await this.evaluateOnPage(tab, expression) if (result === true) return true await new Promise((resolve) => setTimeout(resolve, 250)) } return false } /** Agent 逃生舱:在页面上下文执行自写 JS(有长度上限,结果 JSON 化) */ async evaluate(script: string, tabId?: string): Promise { if (!script.trim()) throw new Error('脚本不能为空。') if (script.length > MAX_SCRIPT_CHARS) throw new Error(`脚本不能超过 ${MAX_SCRIPT_CHARS} 字符。`) this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) return this.runAgentOp(tab, async () => { const result = await this.evaluateOnPage(tab, script, true) this.trace(tab, 'js', `执行页面脚本 ${script.length} 字符`) return result }) } private async evaluateOnPage(tab: TabRecord, expression: string, userGesture = false): Promise { const response = await this.cdp(tab, 'Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true, userGesture }) const exception = response.exceptionDetails as { text?: string; exception?: { description?: string } } | undefined if (exception) { throw new Error(`页面脚本执行出错: ${exception.exception?.description || exception.text || '未知错误'}`) } const result = response.result as { type?: string; value?: unknown; description?: string } | undefined if (!result) return null if ('value' in result) return result.value ?? null return { type: result.type ?? 'unknown', description: result.description ?? null } } async screenshot(tabId?: string): Promise<{ tabId: string; url: string; title: string; dataUrl: string }> { this.assertRiskAcknowledged() const tab = this.getAgentTab(tabId) return this.runAgentOp(tab, async () => { const image = await tab.view.webContents.capturePage() // 记录 像素/DIP 比例,browser_click_point 换算截图坐标用(capturePage 按显示 DPR 放大) const bounds = tab.view.getBounds() if (bounds.width > 0) tab.lastShotScale = image.getSize().width / bounds.width this.trace(tab, 'screenshot', '截取当前页面') return { tabId: tab.tabId, url: tab.url, title: tab.title, dataUrl: image.toDataURL() } }) } /** 渲染层「冻结帧」用:菜单/浮层需要暂时隐藏原生视图前,截当前画面做占位, * 视觉上等于菜单浮在页面之上(DOM 层级永远低于原生视图,菜单弹出必须先让位) */ async captureTabDataUrl(tabId?: string): Promise { const tab = tabId ? this.tabs.get(tabId) : this.activeTabId ? this.tabs.get(this.activeTabId) : null if (!tab) throw new Error('标签页不存在,无法截取画面。') const image = await tab.view.webContents.capturePage() return image.toDataURL() } } const MAX_BROWSER_OBSERVE_DEPTH_FALLBACK = 400 let controllerInstance: BrowserController | null = null /** 惰性单例:首次取用时才创建(session 相关 API 需在 app ready 后调用) */ export function getBrowserController(): BrowserController { if (!controllerInstance) controllerInstance = new BrowserController() return controllerInstance }