1238 lines
52 KiB
TypeScript
1238 lines
52 KiB
TypeScript
import { getLogger } from '../logger'
|
||
const log = getLogger('browser')
|
||
|
||
import { app, BrowserWindow, session as electronSession, WebContentsView, type Rectangle, type Session } from 'electron'
|
||
import { existsSync, readFileSync, writeFileSync } from 'node:fs'
|
||
import { join } from 'node:path'
|
||
import type { BrowserImportInfo, BrowserTabInfo, BrowserUiState } from '@shared/types'
|
||
import {
|
||
BrowserCdpTimeoutError,
|
||
BROWSER_OBSERVE_TIMEOUT_MS,
|
||
resolveBrowserObserveAxDepth,
|
||
withBrowserCdpTimeout,
|
||
type CdpResponse
|
||
} from './browser-cdp'
|
||
import {
|
||
assertSafeBrowserUrl,
|
||
browserObservationNameLimit,
|
||
MAX_BROWSER_FIND_TEXT_RESULTS,
|
||
NAMELESS_ALLOWED_ROLES,
|
||
prioritizeBrowserObservationCandidates,
|
||
resolveBrowserDestination,
|
||
resolveBrowserObserveMaxElements
|
||
} from './browser-policy'
|
||
|
||
/**
|
||
* 内置受管浏览器控制器。
|
||
*
|
||
* 架构（参考 Proma 的成熟实现，适配本应用单窗口/单聊天会话形态）：
|
||
* - 每个标签页 = 一个原生 WebContentsView，通过 contentView.addChildView 呈现；
|
||
* 渲染层 BrowserSlot 量出占位矩形，经 IPC 同步边界（原生视图天然盖在 DOM 之上）。
|
||
* - 页面观察/操作全部走 CDP（webContents.debugger）：AX tree 快照 + ref 代际机制
|
||
* + Input.dispatch* 真实输入事件；不在页面里执行模拟点击。
|
||
* - 登录态隔离在 persist:aso-browser 分区；本地浏览器配置导入由 browser-profile.ts
|
||
* 直接迁移分区目录数据（Cookies/Local Storage + 密钥移植）。
|
||
*/
|
||
|
||
const PARTITION = 'persist:aso-browser'
|
||
// 新标签页默认地址：google.com 在目标市场（国内）不可达，新开标签会长时间白屏，
|
||
// 误以为浏览器故障；用可稳定访问的导航页
|
||
const NEW_TAB_URL = 'https://www.baidu.com/'
|
||
const MAX_TABS = 12
|
||
const MAX_FILL_CHARS = 10_000
|
||
const MAX_SCRIPT_CHARS = 20_000
|
||
const MAX_EXTRACT_CHARS = 50_000
|
||
|
||
interface RefEntry {
|
||
backendNodeId: number
|
||
generation: number
|
||
label: string
|
||
editable: boolean
|
||
}
|
||
|
||
interface TabRecord {
|
||
tabId: string
|
||
view: WebContentsView
|
||
createdAt: number
|
||
url: string
|
||
title: string
|
||
loading: boolean
|
||
/** ref 表与代际号：每次观察/导航后旧 ref 失效 */
|
||
refs: Map<string, RefEntry>
|
||
generation: number
|
||
/** 同一 tab 的 Agent 操作串行化，避免 CDP 命令交错 */
|
||
commandTail: Promise<unknown>
|
||
lastLayoutRevision: number
|
||
attached: boolean
|
||
/** 最近一次截图的 像素/DIP 比例（browser_click_point 换算截图坐标用） */
|
||
lastShotScale: number
|
||
}
|
||
|
||
interface BrowserSettings {
|
||
riskAcknowledged: boolean
|
||
imported?: BrowserImportInfo | null
|
||
}
|
||
|
||
export interface BrowserObservation {
|
||
tabId: string
|
||
url: string
|
||
title: string
|
||
generation: number
|
||
elements: Array<{ ref: string; role: string; name: string; editable: boolean }>
|
||
}
|
||
|
||
/** browser_find_text 的候选描述。rect 为视口 CSS 像素（iframe 内元素已换算为顶层页面坐标） */
|
||
export interface BrowserFindTextCandidate {
|
||
index: number
|
||
tagName: string
|
||
text: string
|
||
rect: { x: number; y: number; w: number; h: number }
|
||
context: string
|
||
/** true = 位于 iframe 内（不能直接注册 ref 点击，需改用坐标点击） */
|
||
iframe?: boolean
|
||
}
|
||
|
||
export interface BrowserFindTextResult extends BrowserObservation {
|
||
matched: number
|
||
picked?: number
|
||
candidates?: BrowserFindTextCandidate[]
|
||
}
|
||
|
||
export interface BrowserWaitCondition {
|
||
kind: 'url' | 'text' | 'selector'
|
||
value: string
|
||
}
|
||
|
||
function textValue(v: unknown): string {
|
||
if (v && typeof v === 'object') {
|
||
const value = (v as { value?: unknown }).value
|
||
if (typeof value === 'string') return value
|
||
}
|
||
return ''
|
||
}
|
||
|
||
function isEditableAxNode(node: Record<string, unknown>): boolean {
|
||
const role = textValue(node.role).toLowerCase()
|
||
if (role === 'textfield' || role === 'textbox' || role === 'searchbox') return true
|
||
const props = Array.isArray(node.properties) ? (node.properties as Array<Record<string, unknown>>) : []
|
||
for (const p of props) {
|
||
if (String(p.name) === 'editable') {
|
||
const inner = p.value as { value?: unknown } | undefined
|
||
if (inner && typeof inner.value === 'boolean') return inner.value
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
/** 导航键 → CDP key 事件参数（rawKeyDown + VK code 才会触发默认行为：滚动/提交/焦点移动） */
|
||
const KEY_EVENTS: Record<string, { key: string; code: string; vk: number }> = {
|
||
enter: { key: 'Enter', code: 'Enter', vk: 13 },
|
||
tab: { key: 'Tab', code: 'Tab', vk: 9 },
|
||
escape: { key: 'Escape', code: 'Escape', vk: 27 },
|
||
backspace: { key: 'Backspace', code: 'Backspace', vk: 8 },
|
||
delete: { key: 'Delete', code: 'Delete', vk: 46 },
|
||
pageup: { key: 'PageUp', code: 'PageUp', vk: 33 },
|
||
pagedown: { key: 'PageDown', code: 'PageDown', vk: 34 },
|
||
home: { key: 'Home', code: 'Home', vk: 36 },
|
||
end: { key: 'End', code: 'End', vk: 35 },
|
||
arrowup: { key: 'ArrowUp', code: 'ArrowUp', vk: 38 },
|
||
arrowdown: { key: 'ArrowDown', code: 'ArrowDown', vk: 40 },
|
||
arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft', vk: 37 },
|
||
arrowright: { key: 'ArrowRight', code: 'ArrowRight', vk: 39 },
|
||
space: { key: ' ', code: 'Space', vk: 32 }
|
||
}
|
||
|
||
/** 页面内文本/选择器查找器（宿主固定函数，参数经 JSON 注入，非任意脚本）。
|
||
* 遍历主文档 + 开放 shadow DOM + 同源 iframe（深度受限），按元素自身直接可见文本或
|
||
* CSS 选择器匹配。返回纯可序列化数据（results 元数据 + diag 诊断 + pick 模式打标结果）。
|
||
* 背景：纯 div/span + JS 点击的元素（商品规格等）不在 AX 快照里，ref 体系够不到它们。
|
||
* 注意：结果回传只用 returnByValue 序列化——Runtime.callFunctionOn 跨命令持有
|
||
* RemoteObject 引用在 Electron debugger 下不可靠（this 绑定丢失，恒返回 undefined）。 */
|
||
const FIND_TEXT_FINDER = String.raw`function(input) {
|
||
const norm = (s) => (s || '').replace(/\s+/g, ' ').trim()
|
||
const results = []
|
||
const docs = []
|
||
let visitedTotal = 0
|
||
const collectDocs = (doc, depth) => {
|
||
if (!doc || docs.indexOf(doc) >= 0 || depth > 3) return
|
||
docs.push(doc)
|
||
let frames = []
|
||
try { frames = Array.prototype.slice.call(doc.querySelectorAll('iframe')) } catch (err) {}
|
||
for (const frame of frames) {
|
||
try { collectDocs(frame.contentDocument, depth + 1) } catch (err) {}
|
||
}
|
||
}
|
||
collectDocs(document, 0)
|
||
if (input.selector) {
|
||
try { document.querySelector(input.selector) } catch (err) {
|
||
return { error: '选择器语法非法' }
|
||
}
|
||
}
|
||
const ownText = (el) => {
|
||
let text = ''
|
||
for (let node = el.firstChild; node; node = node.nextSibling) {
|
||
if (node.nodeType === 3) text += node.nodeValue
|
||
}
|
||
return norm(text)
|
||
}
|
||
const contextOf = (el) => {
|
||
let parent = el.parentElement
|
||
while (parent) {
|
||
const text = norm(parent.textContent)
|
||
if (text) return text.slice(0, 60)
|
||
parent = parent.parentElement
|
||
}
|
||
return ''
|
||
}
|
||
const visible = (el) => {
|
||
try {
|
||
const rect = el.getBoundingClientRect()
|
||
if (rect.width < 1 || rect.height < 1) return false
|
||
const view = el.ownerDocument.defaultView
|
||
if (!view) return false
|
||
const style = view.getComputedStyle(el)
|
||
return style.visibility !== 'hidden' && style.pointerEvents !== 'none'
|
||
} catch (err) {
|
||
return false
|
||
}
|
||
}
|
||
const toTopCoords = (el) => {
|
||
let doc = el.ownerDocument
|
||
let x = 0
|
||
let y = 0
|
||
let guard = 0
|
||
while (doc && doc !== document && guard++ < 5) {
|
||
const findHost = (d) => {
|
||
let frames = []
|
||
try { frames = Array.prototype.slice.call(d.querySelectorAll('iframe')) } catch (err) {}
|
||
for (const frame of frames) {
|
||
if (frame.contentDocument === doc) return frame
|
||
const nested = findHost(frame.contentDocument)
|
||
if (nested) return nested
|
||
}
|
||
return null
|
||
}
|
||
const host = findHost(document)
|
||
if (!host) break
|
||
const hostRect = host.getBoundingClientRect()
|
||
x += hostRect.x
|
||
y += hostRect.y
|
||
doc = host.ownerDocument
|
||
}
|
||
return { x, y }
|
||
}
|
||
const hits = []
|
||
const walk = (root) => {
|
||
let iter = null
|
||
try { iter = root.createTreeWalker(root, NodeFilter.SHOW_ELEMENT) } catch (err) { return }
|
||
let visited = 0
|
||
let node
|
||
while ((node = iter.nextNode())) {
|
||
if (++visited > 20000) break
|
||
visitedTotal++
|
||
const el = node
|
||
if (el.shadowRoot) walk(el.shadowRoot)
|
||
let hit = false
|
||
let label = ''
|
||
if (input.selector) {
|
||
try { hit = el.matches(input.selector) } catch (err) { hit = false }
|
||
if (hit) {
|
||
label = ownText(el)
|
||
if (!label) {
|
||
try { label = norm(el.getAttribute('aria-label') || el.getAttribute('title') || '') } catch (err) {}
|
||
}
|
||
}
|
||
} else {
|
||
label = ownText(el)
|
||
if (label) hit = input.contains ? label.indexOf(input.text) >= 0 : label === input.text
|
||
}
|
||
if (hit && visible(el)) hits.push({ el, label })
|
||
}
|
||
}
|
||
for (const doc of docs) {
|
||
if (doc.documentElement) walk(doc)
|
||
}
|
||
if (input.pickIndex !== undefined && input.pickIndex !== null) {
|
||
const hit = hits[input.pickIndex - 1]
|
||
if (!hit) return { ok: false, diag: { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href) } }
|
||
hit.el.setAttribute('data-aso-find', input.tagValue)
|
||
const rect = hit.el.getBoundingClientRect()
|
||
const offset = hit.el.ownerDocument === document ? { x: 0, y: 0 } : toTopCoords(hit.el)
|
||
return { ok: true, tagged: hit.label.slice(0, 80), rect: { x: Math.round(rect.x + offset.x), y: Math.round(rect.y + offset.y) }, diag: { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href) } }
|
||
}
|
||
for (const hit of hits.slice(0, input.limit)) {
|
||
const rect = hit.el.getBoundingClientRect()
|
||
const offset = hit.el.ownerDocument === document ? { x: 0, y: 0 } : toTopCoords(hit.el)
|
||
results.push({
|
||
index: results.length + 1,
|
||
tagName: hit.el.tagName.toLowerCase(),
|
||
text: hit.label.slice(0, 80),
|
||
rect: { x: Math.round(rect.x + offset.x), y: Math.round(rect.y + offset.y), w: Math.round(rect.width), h: Math.round(rect.height) },
|
||
context: contextOf(hit.el),
|
||
iframe: hit.el.ownerDocument !== document
|
||
})
|
||
}
|
||
const diag = { docs: docs.length, visited: visitedTotal, hits: hits.length, url: String(location.href), readyState: document.readyState, nodeFilter: typeof NodeFilter }
|
||
return { results, diag }
|
||
}`
|
||
|
||
export class BrowserController {
|
||
private owner: BrowserWindow | null = null
|
||
private browserSession: Session
|
||
private tabs = new Map<string, TabRecord>()
|
||
private activeTabId: string | null = null
|
||
private agentTabId: string | null = null
|
||
private listeners = new Set<(state: BrowserUiState) => void>()
|
||
private traceText: string | null = null
|
||
private settings: BrowserSettings = { riskAcknowledged: false }
|
||
/** 最近关闭的页签（新的在前，供「最近关闭的标签页」恢复） */
|
||
private closedTabs: Array<{ title: string; url: string; closedAt: number }> = []
|
||
|
||
constructor() {
|
||
this.browserSession = electronSession.fromPartition(PARTITION)
|
||
// 诚实的 UA：保留 Chromium token 保证站点兼容，去掉 Electron 标识
|
||
this.browserSession.setUserAgent(
|
||
this.browserSession.getUserAgent().replace(/Electron\/[\d.]+/, `InkreachAgent/${app.getVersion()}`)
|
||
)
|
||
// 网页权限请求（摄像头/定位等）默认拒绝
|
||
this.browserSession.setPermissionRequestHandler((_wc, _permission, callback) => callback(false))
|
||
// 下载留在受管浏览器内，落到系统下载目录（重名自动加序号）
|
||
this.browserSession.on('will-download', (_event, item) => {
|
||
const dir = app.getPath('downloads')
|
||
const base = item.getFilename() || 'download'
|
||
const dot = base.lastIndexOf('.')
|
||
const stem = dot > 0 ? base.slice(0, dot) : base
|
||
const ext = dot > 0 ? base.slice(dot) : ''
|
||
let target = join(dir, base)
|
||
for (let i = 1; existsSync(target) && i < 100; i++) {
|
||
target = join(dir, `${stem} (${i})${ext}`)
|
||
}
|
||
item.setSavePath(target)
|
||
})
|
||
this.settings = this.loadSettings()
|
||
}
|
||
|
||
// ---- 设置持久化 ----
|
||
|
||
private get settingsFile(): string {
|
||
return join(app.getPath('userData'), 'aso-browser.json')
|
||
}
|
||
|
||
private loadSettings(): BrowserSettings {
|
||
try {
|
||
const raw = JSON.parse(readFileSync(this.settingsFile, 'utf8')) as Partial<BrowserSettings>
|
||
return {
|
||
riskAcknowledged: raw.riskAcknowledged === true,
|
||
imported: raw.imported ?? null
|
||
}
|
||
} catch {
|
||
return { riskAcknowledged: false }
|
||
}
|
||
}
|
||
|
||
private saveSettings(): void {
|
||
try {
|
||
writeFileSync(this.settingsFile, JSON.stringify(this.settings, null, 2), 'utf8')
|
||
} catch (err) {
|
||
log.error('save settings failed:', { err: err })
|
||
}
|
||
}
|
||
|
||
/** 配置导入完成后由 browser-profile.ts 回写摘要 */
|
||
setImportedInfo(info: BrowserImportInfo | null): void {
|
||
this.settings.imported = info
|
||
this.saveSettings()
|
||
this.emitState()
|
||
}
|
||
|
||
get importedInfo(): BrowserImportInfo | null {
|
||
return this.settings.imported ?? null
|
||
}
|
||
|
||
/** 当前是否有活跃标签页（导入配置前必须没有） */
|
||
get hasTabs(): boolean {
|
||
return this.tabs.size > 0
|
||
}
|
||
|
||
// ---- 窗口与状态广播 ----
|
||
|
||
setOwnerWindow(win: BrowserWindow): void {
|
||
this.owner = win
|
||
}
|
||
|
||
handleWindowClosed(): void {
|
||
for (const tab of this.tabs.values()) {
|
||
try {
|
||
if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach()
|
||
} catch {
|
||
// 已销毁
|
||
}
|
||
try {
|
||
tab.view.setVisible(false)
|
||
} catch {
|
||
// 已销毁
|
||
}
|
||
}
|
||
this.tabs.clear()
|
||
this.activeTabId = null
|
||
this.agentTabId = null
|
||
this.owner = null
|
||
}
|
||
|
||
onState(fn: (state: BrowserUiState) => void): () => void {
|
||
this.listeners.add(fn)
|
||
return () => this.listeners.delete(fn)
|
||
}
|
||
|
||
private buildState(): BrowserUiState {
|
||
const tabs: BrowserTabInfo[] = [...this.tabs.values()]
|
||
.sort((a, b) => a.createdAt - b.createdAt)
|
||
.map((t) => ({ tabId: t.tabId, url: t.url, title: t.title, loading: t.loading }))
|
||
return {
|
||
open: tabs.length > 0,
|
||
riskAcknowledged: this.settings.riskAcknowledged,
|
||
tabs,
|
||
activeTabId: this.activeTabId,
|
||
recentClosed: this.closedTabs.map(({ title, url }) => ({ title, url })),
|
||
trace: this.traceText,
|
||
imported: this.settings.imported ?? null
|
||
}
|
||
}
|
||
|
||
getState(): BrowserUiState {
|
||
return this.buildState()
|
||
}
|
||
|
||
private emitState(): void {
|
||
const state = this.buildState()
|
||
for (const fn of this.listeners) {
|
||
try {
|
||
fn(state)
|
||
} catch (err) {
|
||
log.error('state listener error:', { err: err })
|
||
}
|
||
}
|
||
}
|
||
|
||
private trace(tab: TabRecord, action: string, detail: string): void {
|
||
this.traceText = `[${action}] ${detail}`
|
||
void tab
|
||
this.emitState()
|
||
}
|
||
|
||
// ---- 风险告知门禁 ----
|
||
|
||
acknowledgeRisk(): void {
|
||
this.settings.riskAcknowledged = true
|
||
this.saveSettings()
|
||
this.emitState()
|
||
}
|
||
|
||
private assertRiskAcknowledged(): void {
|
||
if (!this.settings.riskAcknowledged) {
|
||
throw new Error('内置浏览器首次使用尚未确认风险告知：请在应用右侧浏览器面板中阅读并点击「我已了解，开始使用」后重试。')
|
||
}
|
||
}
|
||
|
||
// ---- 会话与标签页 ----
|
||
|
||
private ensureBrowser(): TabRecord {
|
||
if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪，无法使用内置浏览器。')
|
||
if (this.tabs.size === 0) this.createTab(NEW_TAB_URL)
|
||
const active = this.activeTabId ? this.tabs.get(this.activeTabId) : null
|
||
const fallback = this.tabs.values().next().value
|
||
const tab = active ?? fallback
|
||
if (!tab) throw new Error('浏览器标签页不可用。')
|
||
this.emitState()
|
||
return tab
|
||
}
|
||
|
||
private invalidateTabDocument(tab: TabRecord): void {
|
||
tab.generation += 1
|
||
tab.refs.clear()
|
||
}
|
||
|
||
private createTab(initialUrl: string, viewOptions?: Electron.WebContentsViewConstructorOptions): TabRecord {
|
||
if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪，无法创建浏览器标签。')
|
||
if (this.tabs.size >= MAX_TABS) throw new Error(`浏览器标签页已达上限（${MAX_TABS}），请先关闭不需要的标签。`)
|
||
const tabId = `tab-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
||
const view = new WebContentsView(
|
||
viewOptions ?? {
|
||
webPreferences: {
|
||
partition: PARTITION,
|
||
nodeIntegration: false,
|
||
contextIsolation: true,
|
||
sandbox: true,
|
||
webSecurity: true
|
||
}
|
||
}
|
||
)
|
||
const tab: TabRecord = {
|
||
tabId,
|
||
view,
|
||
createdAt: Date.now(),
|
||
url: '',
|
||
title: '新标签页',
|
||
loading: false,
|
||
refs: new Map(),
|
||
generation: 0,
|
||
commandTail: Promise.resolve(),
|
||
lastLayoutRevision: 0,
|
||
attached: false,
|
||
lastShotScale: 1
|
||
}
|
||
view.setVisible(false)
|
||
|
||
// popup 收编为受管同级标签（必须同步返回 webContents，故提前建好 TabRecord）
|
||
view.webContents.setWindowOpenHandler(({ url }) => {
|
||
try {
|
||
assertSafeBrowserUrl(url)
|
||
} catch {
|
||
this.trace(tab, 'popup', `已阻止不安全的新窗口: ${url.slice(0, 80)}`)
|
||
return { action: 'deny' }
|
||
}
|
||
if (this.tabs.get(tabId) !== tab || this.tabs.size >= MAX_TABS) return { action: 'deny' }
|
||
return {
|
||
action: 'allow',
|
||
outlivesOpener: true,
|
||
createWindow: (options) => {
|
||
const popup = this.createTab(url, options)
|
||
this.selectTab(popup.tabId)
|
||
this.trace(popup, 'popup', '网页打开了新标签页')
|
||
return popup.view.webContents
|
||
}
|
||
}
|
||
})
|
||
|
||
const updateNav = (): void => {
|
||
const wc = view.webContents
|
||
if (wc.isDestroyed()) return
|
||
try {
|
||
tab.url = wc.getURL()
|
||
tab.title = wc.getTitle() || tab.url
|
||
tab.loading = wc.isLoading()
|
||
} catch {
|
||
// 竞态时忽略
|
||
}
|
||
this.emitState()
|
||
}
|
||
view.webContents.on('did-start-loading', () => {
|
||
tab.loading = true
|
||
this.invalidateTabDocument(tab)
|
||
this.emitState()
|
||
})
|
||
view.webContents.on('did-stop-loading', updateNav)
|
||
view.webContents.on('page-title-updated', updateNav)
|
||
view.webContents.on('did-navigate', () => {
|
||
this.invalidateTabDocument(tab)
|
||
updateNav()
|
||
})
|
||
view.webContents.on('did-navigate-in-page', () => {
|
||
this.invalidateTabDocument(tab)
|
||
updateNav()
|
||
})
|
||
view.webContents.on('destroyed', () => {
|
||
this.tabs.delete(tabId)
|
||
this.removeView(view)
|
||
if (this.activeTabId === tabId) this.activeTabId = this.tabs.keys().next().value ?? null
|
||
if (this.agentTabId === tabId) this.agentTabId = this.activeTabId
|
||
this.emitState()
|
||
})
|
||
|
||
try {
|
||
view.webContents.debugger.attach('1.3')
|
||
} catch (err) {
|
||
log.warn('CDP attach 失败:', { err: err })
|
||
}
|
||
this.tabs.set(tabId, tab)
|
||
if (!this.activeTabId) this.activeTabId = tabId
|
||
// popup 场景（带 viewOptions）由 Electron 的 createWindow 机制自行导航，避免双重加载
|
||
if (initialUrl && !viewOptions) void view.webContents.loadURL(assertSafeBrowserUrl(initialUrl)).catch(() => undefined)
|
||
return tab
|
||
}
|
||
|
||
private removeView(view: WebContentsView): void {
|
||
try {
|
||
this.owner?.contentView.removeChildView(view)
|
||
} catch {
|
||
// 窗口已销毁
|
||
}
|
||
try {
|
||
view.webContents.close()
|
||
} catch {
|
||
// 已销毁
|
||
}
|
||
}
|
||
|
||
listTabs(): BrowserUiState {
|
||
return this.buildState()
|
||
}
|
||
|
||
newTab(url?: string, byAgent = true): BrowserUiState {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.createTab(url ? resolveBrowserDestination(url) : NEW_TAB_URL)
|
||
this.activeTabId = tab.tabId
|
||
if (byAgent) this.agentTabId = tab.tabId
|
||
this.emitState()
|
||
return this.buildState()
|
||
}
|
||
|
||
selectTab(tabId: string): BrowserUiState {
|
||
const tab = this.tabs.get(tabId)
|
||
if (!tab) throw new Error(`标签页不存在: ${tabId}`)
|
||
this.activeTabId = tabId
|
||
try {
|
||
tab.view.webContents.focus()
|
||
} catch {
|
||
// 页面可能正在加载
|
||
}
|
||
this.emitState()
|
||
return this.buildState()
|
||
}
|
||
|
||
closeTab(tabId?: string): BrowserUiState {
|
||
const id = tabId ?? this.activeTabId
|
||
if (!id) return this.buildState()
|
||
const tab = this.tabs.get(id)
|
||
if (!tab) return this.buildState()
|
||
try {
|
||
if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach()
|
||
} catch {
|
||
// 已销毁
|
||
}
|
||
this.removeView(tab.view)
|
||
this.tabs.delete(id)
|
||
// 记入「最近关闭的标签页」（新的在前，最多 10 条）
|
||
this.closedTabs.unshift({ title: tab.title || tab.url || '新标签页', url: tab.url, closedAt: Date.now() })
|
||
this.closedTabs = this.closedTabs.slice(0, 10)
|
||
if (this.activeTabId === id) this.activeTabId = this.tabs.keys().next().value ?? null
|
||
if (this.agentTabId === id) this.agentTabId = this.activeTabId
|
||
this.emitState()
|
||
return this.buildState()
|
||
}
|
||
|
||
/** 关闭整个受管浏览器（销毁全部标签；导入配置前也会走这里） */
|
||
close(): BrowserUiState {
|
||
for (const tabId of [...this.tabs.keys()]) this.closeTab(tabId)
|
||
this.agentTabId = null
|
||
this.traceText = null
|
||
return this.buildState()
|
||
}
|
||
|
||
// ---- 渲染层布局同步 ----
|
||
|
||
applyLayout(layout: { tabId: string; visible: boolean; bounds: Rectangle; revision: number }): void {
|
||
const tab = this.tabs.get(layout.tabId)
|
||
if (!tab || !this.owner || this.owner.isDestroyed()) return
|
||
if (layout.revision <= tab.lastLayoutRevision) return
|
||
tab.lastLayoutRevision = layout.revision
|
||
try {
|
||
if (layout.visible && layout.bounds.width > 4 && layout.bounds.height > 4) {
|
||
if (!tab.attached) {
|
||
this.owner.contentView.addChildView(tab.view)
|
||
tab.attached = true
|
||
}
|
||
tab.view.setBounds(layout.bounds)
|
||
tab.view.setVisible(true)
|
||
} else {
|
||
tab.view.setVisible(false)
|
||
}
|
||
} catch (err) {
|
||
log.error('apply layout failed:', { err: err })
|
||
}
|
||
}
|
||
|
||
// ---- 用户导航（地址栏/按钮） ----
|
||
|
||
navigateDisplay(input: string): BrowserUiState {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.ensureBrowser()
|
||
void tab.view.webContents.loadURL(resolveBrowserDestination(input)).catch(() => undefined)
|
||
this.trace(tab, 'navigate', `打开 ${input.slice(0, 80)}`)
|
||
return this.buildState()
|
||
}
|
||
|
||
goBack(): BrowserUiState {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.ensureBrowser()
|
||
if (tab.view.webContents.navigationHistory.canGoBack()) tab.view.webContents.navigationHistory.goBack()
|
||
return this.buildState()
|
||
}
|
||
|
||
goForward(): BrowserUiState {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.ensureBrowser()
|
||
if (tab.view.webContents.navigationHistory.canGoForward()) tab.view.webContents.navigationHistory.goForward()
|
||
return this.buildState()
|
||
}
|
||
|
||
reload(): BrowserUiState {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.ensureBrowser()
|
||
tab.view.webContents.reload()
|
||
return this.buildState()
|
||
}
|
||
|
||
// ---- CDP 基础 ----
|
||
|
||
private cdp(tab: TabRecord, method: string, params?: Record<string, unknown>, timeoutMs?: number): Promise<CdpResponse> {
|
||
const debuggerClient = tab.view.webContents.debugger
|
||
if (!debuggerClient.isAttached()) throw new Error('浏览器调试通道不可用，请重新打开该标签页。')
|
||
return withBrowserCdpTimeout(() => debuggerClient.sendCommand(method, params) as Promise<CdpResponse>, method, timeoutMs).catch(
|
||
(error: unknown) => {
|
||
if (error instanceof BrowserCdpTimeoutError) this.recoverDebugger(tab, method)
|
||
throw error
|
||
}
|
||
)
|
||
}
|
||
|
||
/** 超时后重连 debugger，避免一个卡死页面让后续浏览器调用全部失败 */
|
||
private recoverDebugger(tab: TabRecord, method: string): void {
|
||
try {
|
||
const debuggerClient = tab.view.webContents.debugger
|
||
if (debuggerClient.isAttached()) debuggerClient.detach()
|
||
debuggerClient.attach('1.3')
|
||
log.warn(`CDP ${method} timeout, reattached`)
|
||
} catch {
|
||
// 页面已关闭
|
||
}
|
||
}
|
||
|
||
/** Agent 的 tab 内操作串行化（用户操作不排队） */
|
||
private runAgentOp<T>(tab: TabRecord, fn: () => Promise<T>): Promise<T> {
|
||
const next = tab.commandTail.then(fn, fn)
|
||
tab.commandTail = next.catch(() => undefined)
|
||
return next
|
||
}
|
||
|
||
private getAgentTab(tabId?: string): TabRecord {
|
||
if (tabId) {
|
||
const explicit = this.tabs.get(tabId)
|
||
if (!explicit) throw new Error('目标标签页不存在，请先 browser_list_tabs 并用 browser_new_tab/browser_select_tab 选择。')
|
||
return explicit
|
||
}
|
||
const id = this.agentTabId ?? this.activeTabId
|
||
const tab = id ? this.tabs.get(id) : null
|
||
if (tab) return tab
|
||
// Agent 首次操作时还没有任何标签页：自动开一个工作页
|
||
return this.ensureBrowser()
|
||
}
|
||
|
||
private assertCurrentDocument(tab: TabRecord, generation: number): void {
|
||
if (tab.generation !== generation) {
|
||
throw new Error('页面已变化（导航或重渲染），旧元素引用失效：请重新 browser_observe 后再操作。')
|
||
}
|
||
}
|
||
|
||
private resolveRef(tab: TabRecord, ref: string): RefEntry {
|
||
const entry = tab.refs.get(ref)
|
||
if (!entry || entry.generation !== tab.generation) {
|
||
throw new Error('元素引用已失效，请先重新调用 browser_observe。')
|
||
}
|
||
return entry
|
||
}
|
||
|
||
/** AX ref → 视口中心坐标（先滚动到可见再读 box；懒加载列表滚动后坐标会变，不能复用旧值） */
|
||
private async centerForRef(tab: TabRecord, ref: string, generation: number): Promise<{ x: number; y: number }> {
|
||
this.assertCurrentDocument(tab, generation)
|
||
const { backendNodeId } = this.resolveRef(tab, ref)
|
||
await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId })
|
||
this.assertCurrentDocument(tab, generation)
|
||
const box = await this.cdp(tab, 'DOM.getBoxModel', { backendNodeId })
|
||
this.assertCurrentDocument(tab, generation)
|
||
const model = box.model as Record<string, unknown> | undefined
|
||
const quad = Array.isArray(model?.content) ? (model?.content as unknown[]) : []
|
||
if (quad.length < 8 || !quad.every((v) => typeof v === 'number')) {
|
||
throw new Error('目标元素当前不可点击，请重新观察页面。')
|
||
}
|
||
const n = quad as number[]
|
||
return {
|
||
x: (n[0] + n[2] + n[4] + n[6]) / 4,
|
||
y: (n[1] + n[3] + n[5] + n[7]) / 4
|
||
}
|
||
}
|
||
|
||
/** 点击前在页面上短暂高亮目标，用户能看清 Agent 正在操作什么 */
|
||
private async highlightAgentTarget(tab: TabRecord, backendNodeId: number): Promise<void> {
|
||
try {
|
||
const resolved = await this.cdp(tab, 'DOM.resolveNode', { backendNodeId })
|
||
const objectId = (resolved.object as { objectId?: string } | undefined)?.objectId
|
||
if (!objectId) return
|
||
await this.cdp(tab, 'Runtime.callFunctionOn', {
|
||
functionDeclaration:
|
||
'function(){const r=this.getBoundingClientRect();const d=document.createElement("div");' +
|
||
'd.style.cssText=`position:fixed;left:${r.left-2}px;top:${r.top-2}px;width:${r.width+4}px;height:${r.height+4}px;`+' +
|
||
'"outline:2px solid #f59e0b;outline-offset:1px;pointer-events:none;z-index:2147483647;";' +
|
||
'document.documentElement.appendChild(d);setTimeout(()=>d.remove(),700);return true}',
|
||
objectId,
|
||
returnByValue: true
|
||
})
|
||
} catch {
|
||
// 高亮失败不影响操作
|
||
}
|
||
}
|
||
|
||
// ---- Agent 工具实现 ----
|
||
|
||
async navigate(url: string, tabId?: string): Promise<BrowserUiState> {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
this.agentTabId = tab.tabId
|
||
this.activeTabId = tab.tabId
|
||
const destination = resolveBrowserDestination(url)
|
||
await tab.view.webContents.loadURL(destination)
|
||
this.trace(tab, 'navigate', `打开 ${destination.slice(0, 80)}`)
|
||
return this.buildState()
|
||
}
|
||
|
||
private parseAxNodes(response: CdpResponse): Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> {
|
||
const nodes = Array.isArray(response.nodes) ? (response.nodes as Array<Record<string, unknown>>) : []
|
||
const candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> = []
|
||
for (const node of nodes) {
|
||
if (!node) continue
|
||
const backendNodeId = typeof node.backendDOMNodeId === 'number' ? node.backendDOMNodeId : 0
|
||
const role = textValue(node.role).toLowerCase()
|
||
const name = textValue(node.name)
|
||
const editable = isEditableAxNode(node)
|
||
if (!backendNodeId || !role) continue
|
||
if (!name && !editable && !NAMELESS_ALLOWED_ROLES.has(role)) continue
|
||
candidates.push({ backendNodeId, role, name: name.slice(0, browserObservationNameLimit(role)), editable })
|
||
}
|
||
return candidates
|
||
}
|
||
|
||
private buildRefs(tab: TabRecord, candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }>): BrowserObservation['elements'] {
|
||
tab.generation += 1
|
||
tab.refs.clear()
|
||
const elements: BrowserObservation['elements'] = []
|
||
for (const candidate of candidates) {
|
||
const ref = `r${tab.generation}-${elements.length + 1}`
|
||
tab.refs.set(ref, {
|
||
backendNodeId: candidate.backendNodeId,
|
||
generation: tab.generation,
|
||
label: candidate.name ? `${candidate.role}「${candidate.name.slice(0, 80)}」` : candidate.role,
|
||
editable: candidate.editable
|
||
})
|
||
elements.push({ ref, role: candidate.role, name: candidate.name, editable: candidate.editable })
|
||
}
|
||
return elements
|
||
}
|
||
|
||
async observe(tabId?: string, requestedMaxElements?: number): Promise<BrowserObservation> {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
this.agentTabId = tab.tabId
|
||
return this.runAgentOp(tab, async () => {
|
||
const maxElements = resolveBrowserObserveMaxElements(requestedMaxElements)
|
||
const depth = resolveBrowserObserveAxDepth(maxElements)
|
||
const response = await this.cdp(
|
||
tab,
|
||
'Accessibility.getFullAXTree',
|
||
{ depth },
|
||
BROWSER_OBSERVE_TIMEOUT_MS
|
||
)
|
||
const selected = prioritizeBrowserObservationCandidates(this.parseAxNodes(response), maxElements)
|
||
const elements = this.buildRefs(tab, selected)
|
||
this.trace(tab, 'observe', `读取到 ${elements.length}/${maxElements} 个元素（AX 深度 ${depth}）`)
|
||
return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements }
|
||
})
|
||
}
|
||
|
||
async find(
|
||
query: { role?: string; name?: string; exact?: boolean; maxResults?: number },
|
||
tabId?: string
|
||
): Promise<BrowserObservation> {
|
||
const role = query.role?.trim().toLowerCase()
|
||
const name = query.name?.trim().toLowerCase()
|
||
const maxResults = query.maxResults ?? 20
|
||
if (!role && !name) throw new Error('语义定位至少需要 role 或 name。')
|
||
if (!Number.isInteger(maxResults) || maxResults < 1 || maxResults > 50) throw new Error('maxResults 必须是 1 到 50 的整数。')
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
this.agentTabId = tab.tabId
|
||
return this.runAgentOp(tab, async () => {
|
||
// 过滤后再套观察配额：Find 能找到被紧凑 Observe 截掉的目标
|
||
const response = await this.cdp(
|
||
tab,
|
||
'Accessibility.getFullAXTree',
|
||
{ depth: resolveBrowserObserveAxDepth(MAX_BROWSER_OBSERVE_DEPTH_FALLBACK) },
|
||
BROWSER_OBSERVE_TIMEOUT_MS
|
||
)
|
||
const candidates = this.parseAxNodes(response).filter((candidate) => {
|
||
const roleMatches = !role || candidate.role === role
|
||
const candidateName = candidate.name.toLowerCase()
|
||
const nameMatches = !name || (query.exact ? candidateName === name : candidateName.includes(name))
|
||
return roleMatches && nameMatches
|
||
})
|
||
const elements = this.buildRefs(tab, candidates.slice(0, maxResults))
|
||
this.trace(tab, 'find', `语义定位到 ${elements.length} 个元素（${role ?? name}）`)
|
||
return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements }
|
||
})
|
||
}
|
||
|
||
/** 按可见文本/CSS 选择器在 DOM 中直查（不依赖无障碍树）：命中后注册为普通 ref，
|
||
* browser_click 等后续操作照常使用。多命中且未指定 index 时只返回候选列表不发 ref；
|
||
* 仅在真正注册 ref 时才作废旧 ref（与 observe/find 的失效语义一致）。 */
|
||
async findText(
|
||
query: { text?: string; selector?: string; contains?: boolean; index?: number },
|
||
tabId?: string
|
||
): Promise<BrowserFindTextResult> {
|
||
const text = query.text?.trim().slice(0, 200) ?? ''
|
||
const selector = query.selector?.trim().slice(0, 500) ?? ''
|
||
if ((text ? 1 : 0) + (selector ? 1 : 0) !== 1) throw new Error('text 与 selector 必须且只能提供一个。')
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
this.agentTabId = tab.tabId
|
||
return this.runAgentOp(tab, async () => {
|
||
const baseInput = { text: text || undefined, selector: selector || undefined, contains: query.contains === true, limit: MAX_BROWSER_FIND_TEXT_RESULTS }
|
||
const payload = await this.evaluateOnPage(tab, `((${FIND_TEXT_FINDER})(${JSON.stringify(baseInput)}))`) as
|
||
| { error?: string; results?: unknown[]; diag?: unknown }
|
||
| null
|
||
if (!payload || typeof payload !== 'object') throw new Error('页面定位结果异常，请重试。')
|
||
if (payload.error) throw new Error(`定位失败：${payload.error}`)
|
||
const diag = payload.diag
|
||
const candidates = (Array.isArray(payload.results) ? payload.results : []).filter(
|
||
(c): c is BrowserFindTextCandidate =>
|
||
!!c && typeof c === 'object' && typeof (c as BrowserFindTextCandidate).tagName === 'string'
|
||
)
|
||
if (candidates.length === 0) {
|
||
this.trace(tab, 'find_text', `未找到 ${text || selector}`)
|
||
return {
|
||
tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation,
|
||
matched: 0, elements: [], candidates: [], ...(diag !== undefined ? { diag } : {})
|
||
}
|
||
}
|
||
const picked = query.index ?? 1
|
||
if (!Number.isInteger(picked) || picked < 1 || picked > candidates.length) {
|
||
throw new Error(`index 必须是 1 到 ${candidates.length} 之间的整数。`)
|
||
}
|
||
if (candidates.length > 1 && query.index === undefined) {
|
||
this.trace(tab, 'find_text', `「${text || selector}」命中 ${candidates.length} 处，等待指定 index`)
|
||
return {
|
||
tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation,
|
||
matched: candidates.length, elements: [], candidates, ...(diag !== undefined ? { diag } : {})
|
||
}
|
||
}
|
||
const candidate = candidates[picked - 1]
|
||
if (candidate.iframe === true) {
|
||
throw new Error('目标元素位于 iframe 内，无法直接注册 ref：请基于 browser_screenshot 用 browser_click_point 点击（候选列表 rect 已换算为页面顶层坐标）。')
|
||
}
|
||
// 选定阶段：给第 N 个命中打临时 data 属性，再经 DOM 域换成 backendNodeId。
|
||
// （DOM.querySelector 不穿 shadow DOM，shadow 内的命中会在下一步报"未找到"，
|
||
// 报错信息会引导改用坐标点击）
|
||
const tagValue = Math.random().toString(36).slice(2, 10)
|
||
const pickPayload = await this.evaluateOnPage(
|
||
tab,
|
||
`((${FIND_TEXT_FINDER})(${JSON.stringify({ ...baseInput, pickIndex: picked, tagValue })}))`
|
||
) as { ok?: boolean; tagged?: string } | null
|
||
if (!pickPayload || pickPayload.ok !== true) {
|
||
throw new Error('页面已变化，目标元素不存在了，请重新调用 browser_find_text。')
|
||
}
|
||
const documentResponse = await this.cdp(tab, 'DOM.getDocument', { depth: 0 })
|
||
const rootId = (documentResponse.root as { nodeId?: number } | undefined)?.nodeId
|
||
if (typeof rootId !== 'number' || rootId <= 0) throw new Error('无法访问页面文档，请重试。')
|
||
const queryResponse = await this.cdp(tab, 'DOM.querySelector', {
|
||
nodeId: rootId,
|
||
selector: `[data-aso-find="${tagValue}"]`
|
||
})
|
||
const taggedNodeId = (queryResponse as { nodeId?: number }).nodeId
|
||
if (typeof taggedNodeId !== 'number' || taggedNodeId === 0) {
|
||
throw new Error('目标元素无法转换为可点击引用（可能位于 shadow DOM 内），请改用 browser_screenshot + browser_click_point。')
|
||
}
|
||
const describeResponse = await this.cdp(tab, 'DOM.describeNode', { nodeId: taggedNodeId })
|
||
const backendNodeId = (describeResponse.node as { backendNodeId?: number } | undefined)?.backendNodeId
|
||
await this.cdp(tab, 'DOM.removeAttribute', { nodeId: taggedNodeId, name: 'data-aso-find' }).catch(() => undefined)
|
||
if (typeof backendNodeId !== 'number' || backendNodeId <= 0) throw new Error('无法解析目标元素，请重新定位。')
|
||
const elements = this.buildRefs(tab, [
|
||
{ backendNodeId, role: candidate.tagName, name: candidate.text, editable: false }
|
||
])
|
||
this.trace(tab, 'find_text', `定位「${(candidate.text || candidate.tagName).slice(0, 40)}」→ ${elements[0]?.ref}（共 ${candidates.length} 处）`)
|
||
return {
|
||
tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation,
|
||
matched: candidates.length, picked,
|
||
elements,
|
||
candidates: candidates.length > 1 ? candidates : undefined
|
||
}
|
||
})
|
||
}
|
||
|
||
private async clickRef(tab: TabRecord, ref: string, generation: number): Promise<RefEntry> {
|
||
const target = this.resolveRef(tab, ref)
|
||
const { x, y } = await this.centerForRef(tab, ref, generation)
|
||
this.assertCurrentDocument(tab, generation)
|
||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 })
|
||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 })
|
||
return target
|
||
}
|
||
|
||
async click(ref: string, tabId?: string): Promise<BrowserUiState> {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
return this.runAgentOp(tab, async () => {
|
||
const generation = tab.generation
|
||
const target = await this.clickRef(tab, ref, generation)
|
||
this.trace(tab, 'click', `点击 ${target.label}`)
|
||
return this.buildState()
|
||
})
|
||
}
|
||
|
||
/** 点击 + 等待合并为一次串行操作（已知点击后预期状态时优先使用） */
|
||
async act(ref: string, waitFor: BrowserWaitCondition | undefined, timeoutMs = 10_000, tabId?: string): Promise<{ state: BrowserUiState; matched: boolean }> {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
return this.runAgentOp(tab, async () => {
|
||
const generation = tab.generation
|
||
const target = await this.clickRef(tab, ref, generation)
|
||
this.trace(tab, 'click', `点击 ${target.label}`)
|
||
if (!waitFor) return { state: this.buildState(), matched: true }
|
||
const matched = await this.waitForInternal(tab, waitFor, timeoutMs)
|
||
return { state: this.buildState(), matched }
|
||
})
|
||
}
|
||
|
||
async hover(ref: string, tabId?: string): Promise<BrowserUiState> {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
return this.runAgentOp(tab, async () => {
|
||
const generation = tab.generation
|
||
const target = this.resolveRef(tab, ref)
|
||
const { x, y } = await this.centerForRef(tab, ref, generation)
|
||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||
this.assertCurrentDocument(tab, generation)
|
||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y })
|
||
this.trace(tab, 'hover', `悬停 ${target.label}`)
|
||
return this.buildState()
|
||
})
|
||
}
|
||
|
||
/** 截图坐标兜底点击：坐标以最近一次 browser_screenshot 图像的像素为准（按截图比例换算 DIP）。
|
||
* 仅供 observe/find/find_text 都无法定位时使用（canvas、封闭 shadow DOM、混淆页面）。 */
|
||
async clickPoint(x: number, y: number, tabId?: string): Promise<BrowserUiState> {
|
||
if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0) throw new Error('坐标必须是非负数。')
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
this.agentTabId = tab.tabId
|
||
return this.runAgentOp(tab, async () => {
|
||
const bounds = tab.view.getBounds()
|
||
if (bounds.width <= 0 || bounds.height <= 0) throw new Error('浏览器面板当前不可见，无法按坐标点击。')
|
||
const scale = tab.lastShotScale > 0 ? tab.lastShotScale : 1
|
||
const dipX = x / scale
|
||
const dipY = y / scale
|
||
if (dipX >= bounds.width || dipY >= bounds.height) {
|
||
throw new Error(
|
||
`坐标越界：换算后 (${Math.round(dipX)}, ${Math.round(dipY)}) 超出页面 ${Math.round(bounds.width)}×${Math.round(bounds.height)}，请基于最新截图重新取坐标。`
|
||
)
|
||
}
|
||
// 点击位置标记（宿主固定脚本，纯视觉反馈）
|
||
const marker =
|
||
'((p)=>{try{const d=document.createElement("div");d.style.cssText="position:fixed;left:"+(p.x-9)+"px;top:"+(p.y-9)+' +
|
||
'"px;width:18px;height:18px;border:2px solid #f59e0b;border-radius:50%;pointer-events:none;z-index:2147483647;box-sizing:border-box";' +
|
||
'document.documentElement.appendChild(d);setTimeout(()=>d.remove(),700);return true}catch(e){return false}})(' +
|
||
JSON.stringify({ x: dipX, y: dipY }) +
|
||
')'
|
||
await this.cdp(tab, 'Runtime.evaluate', { expression: marker, returnByValue: true }).catch(() => undefined)
|
||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: dipX, y: dipY })
|
||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: dipX, y: dipY, button: 'left', clickCount: 1 })
|
||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: dipX, y: dipY, button: 'left', clickCount: 1 })
|
||
this.trace(tab, 'click_point', `点击截图坐标 (${Math.round(x)}, ${Math.round(y)})`)
|
||
return this.buildState()
|
||
})
|
||
}
|
||
|
||
async fill(ref: string, text: string, tabId?: string): Promise<BrowserUiState> {
|
||
if (text.length > MAX_FILL_CHARS) throw new Error(`单次输入不能超过 ${MAX_FILL_CHARS} 个字符。`)
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
return this.runAgentOp(tab, async () => {
|
||
const generation = tab.generation
|
||
const target = this.resolveRef(tab, ref)
|
||
if (!target.editable) throw new Error('目标元素不是可编辑字段，请重新观察后选择 input、textarea 或 contenteditable。')
|
||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||
await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backendNodeId })
|
||
this.assertCurrentDocument(tab, generation)
|
||
await this.cdp(tab, 'DOM.focus', { backendNodeId: target.backendNodeId })
|
||
this.assertCurrentDocument(tab, generation)
|
||
// 全选（Ctrl/Cmd+A）后 insertText 整段替换，不逐字符模拟
|
||
const selectAllModifier = process.platform === 'darwin' ? 4 : 2
|
||
await this.cdp(tab, 'Input.dispatchKeyEvent', {
|
||
type: 'keyDown',
|
||
key: 'a',
|
||
code: 'KeyA',
|
||
windowsVirtualKeyCode: 65,
|
||
modifiers: selectAllModifier
|
||
})
|
||
await this.cdp(tab, 'Input.dispatchKeyEvent', {
|
||
type: 'keyUp',
|
||
key: 'a',
|
||
code: 'KeyA',
|
||
windowsVirtualKeyCode: 65,
|
||
modifiers: selectAllModifier
|
||
})
|
||
this.assertCurrentDocument(tab, generation)
|
||
await this.cdp(tab, 'Input.insertText', { text })
|
||
this.trace(tab, 'fill', `在 ${target.label} 输入 ${Array.from(text).length} 个字符`)
|
||
return this.buildState()
|
||
})
|
||
}
|
||
|
||
async press(key: string, tabId?: string): Promise<BrowserUiState> {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
return this.runAgentOp(tab, async () => {
|
||
const normalized = key.trim().toLowerCase().replace(/\s+/g, '')
|
||
const keyEvent = KEY_EVENTS[normalized]
|
||
if (keyEvent) {
|
||
const event = { key: keyEvent.key, code: keyEvent.code, windowsVirtualKeyCode: keyEvent.vk }
|
||
await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'rawKeyDown', ...event })
|
||
await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'keyUp', ...event })
|
||
this.trace(tab, 'press', `按下 ${keyEvent.key}`)
|
||
} else {
|
||
await this.cdp(tab, 'Input.insertText', { text: key })
|
||
this.trace(tab, 'press', `输入 ${Array.from(key).length} 个字符`)
|
||
}
|
||
return this.buildState()
|
||
})
|
||
}
|
||
|
||
async scroll(
|
||
input: { deltaY?: number; position?: 'top' | 'bottom'; selector?: string },
|
||
tabId?: string
|
||
): Promise<{ before: { top: number; height: number }; after: { top: number; height: number }; moved: boolean }> {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
return this.runAgentOp(tab, async () => {
|
||
const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : ''
|
||
const position = input.position === 'top' || input.position === 'bottom' ? input.position : undefined
|
||
const deltaY = Math.max(-50_000, Math.min(50_000, Math.round(input.deltaY ?? 600)))
|
||
// 固定表达式：数据由 JSON 注入，不拼接用户脚本
|
||
const expression =
|
||
'((input)=>{const el=input.selector?document.querySelector(input.selector):(document.scrollingElement||document.documentElement);' +
|
||
'if(!el)return {ok:false};const read=()=>({top:Math.round(el.scrollTop),height:Math.round(el.scrollHeight)});' +
|
||
'const before=read();const top=input.position==="top"?0:input.position==="bottom"?Math.max(0,el.scrollHeight-el.clientHeight):el.scrollTop+input.deltaY;' +
|
||
'el.scrollTo({top});const after=read();return {ok:true,before,after}})(' +
|
||
JSON.stringify({ selector, position, deltaY }) +
|
||
')'
|
||
const response = await this.evaluateOnPage(tab, expression)
|
||
const result = (response ?? {}) as { ok?: boolean; before?: { top: number; height: number }; after?: { top: number; height: number } }
|
||
if (!result.ok || !result.before || !result.after) throw new Error('滚动目标不存在或不支持滚动，请检查 selector。')
|
||
this.trace(tab, 'scroll', `滚动页面${selector ? `（${selector.slice(0, 40)}）` : ''}`)
|
||
return { before: result.before, after: result.after, moved: result.after.top !== result.before.top }
|
||
})
|
||
}
|
||
|
||
async extract(
|
||
input: { selector?: string; maxChars?: number },
|
||
tabId?: string
|
||
): Promise<{ url: string; title: string; text: string; truncated: boolean }> {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
return this.runAgentOp(tab, async () => {
|
||
const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : ''
|
||
const maxChars = Math.min(MAX_EXTRACT_CHARS, Math.max(500, Math.round(input.maxChars ?? MAX_EXTRACT_CHARS)))
|
||
const expression =
|
||
'((input)=>{const root=input.selector?document.querySelector(input.selector):document.body;' +
|
||
'if(!root)return null;const text=(root.innerText||"").replace(/\\n{3,}/g,"\\n\\n");' +
|
||
'return {text:text.slice(0,input.maxChars),truncated:text.length>input.maxChars}})(' +
|
||
JSON.stringify({ selector, maxChars }) +
|
||
')'
|
||
const response = await this.evaluateOnPage(tab, expression)
|
||
const result = (response ?? {}) as { text?: string; truncated?: boolean } | null
|
||
if (!result || typeof result.text !== 'string') throw new Error('提取目标不存在，请检查 selector。')
|
||
this.trace(tab, 'extract', `提取正文 ${result.text.length} 字符${selector ? `（${selector.slice(0, 40)}）` : ''}`)
|
||
return { url: tab.url, title: tab.title, text: result.text, truncated: result.truncated === true }
|
||
})
|
||
}
|
||
|
||
async waitFor(condition: BrowserWaitCondition, timeoutMs = 10_000, tabId?: string): Promise<{ matched: boolean; url: string; title: string }> {
|
||
if (!condition.value.trim()) throw new Error('等待条件不能为空。')
|
||
if (!Number.isFinite(timeoutMs) || timeoutMs < 250 || timeoutMs > 30_000) throw new Error('等待超时必须在 250 到 30000 毫秒之间。')
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
return this.runAgentOp(tab, async () => {
|
||
const matched = await this.waitForInternal(tab, condition, timeoutMs)
|
||
this.trace(tab, 'wait', `${matched ? '已满足' : '超时未满足'}${condition.kind} 条件（${condition.value.slice(0, 40)}）`)
|
||
return { matched, url: tab.url, title: tab.title }
|
||
})
|
||
}
|
||
|
||
/** 轮询固定表达式（只读检查，不执行任意 JS） */
|
||
private async waitForInternal(tab: TabRecord, condition: BrowserWaitCondition, timeoutMs: number): Promise<boolean> {
|
||
const startedAt = Date.now()
|
||
const payload = JSON.stringify(condition).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029')
|
||
const expression =
|
||
`(() => { const condition = ${payload}; try { ` +
|
||
"if (condition.kind === 'url') return location.href.includes(condition.value); " +
|
||
"if (condition.kind === 'text') return (document.body?.innerText || '').includes(condition.value); " +
|
||
'return !!document.querySelector(condition.value); } catch { return false; } })()'
|
||
while (Date.now() - startedAt <= timeoutMs) {
|
||
const result = await this.evaluateOnPage(tab, expression)
|
||
if (result === true) return true
|
||
await new Promise((resolve) => setTimeout(resolve, 250))
|
||
}
|
||
return false
|
||
}
|
||
|
||
/** Agent 逃生舱：在页面上下文执行自写 JS（有长度上限，结果 JSON 化） */
|
||
async evaluate(script: string, tabId?: string): Promise<unknown> {
|
||
if (!script.trim()) throw new Error('脚本不能为空。')
|
||
if (script.length > MAX_SCRIPT_CHARS) throw new Error(`脚本不能超过 ${MAX_SCRIPT_CHARS} 字符。`)
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
return this.runAgentOp(tab, async () => {
|
||
const result = await this.evaluateOnPage(tab, script, true)
|
||
this.trace(tab, 'js', `执行页面脚本 ${script.length} 字符`)
|
||
return result
|
||
})
|
||
}
|
||
|
||
private async evaluateOnPage(tab: TabRecord, expression: string, userGesture = false): Promise<unknown> {
|
||
const response = await this.cdp(tab, 'Runtime.evaluate', {
|
||
expression,
|
||
awaitPromise: true,
|
||
returnByValue: true,
|
||
userGesture
|
||
})
|
||
const exception = response.exceptionDetails as { text?: string; exception?: { description?: string } } | undefined
|
||
if (exception) {
|
||
throw new Error(`页面脚本执行出错: ${exception.exception?.description || exception.text || '未知错误'}`)
|
||
}
|
||
const result = response.result as { type?: string; value?: unknown; description?: string } | undefined
|
||
if (!result) return null
|
||
if ('value' in result) return result.value ?? null
|
||
return { type: result.type ?? 'unknown', description: result.description ?? null }
|
||
}
|
||
|
||
async screenshot(tabId?: string): Promise<{ tabId: string; url: string; title: string; dataUrl: string }> {
|
||
this.assertRiskAcknowledged()
|
||
const tab = this.getAgentTab(tabId)
|
||
return this.runAgentOp(tab, async () => {
|
||
const image = await tab.view.webContents.capturePage()
|
||
// 记录 像素/DIP 比例，browser_click_point 换算截图坐标用（capturePage 按显示 DPR 放大）
|
||
const bounds = tab.view.getBounds()
|
||
if (bounds.width > 0) tab.lastShotScale = image.getSize().width / bounds.width
|
||
this.trace(tab, 'screenshot', '截取当前页面')
|
||
return { tabId: tab.tabId, url: tab.url, title: tab.title, dataUrl: image.toDataURL() }
|
||
})
|
||
}
|
||
|
||
/** 渲染层「冻结帧」用：菜单/浮层需要暂时隐藏原生视图前，截当前画面做占位，
|
||
* 视觉上等于菜单浮在页面之上（DOM 层级永远低于原生视图，菜单弹出必须先让位） */
|
||
async captureTabDataUrl(tabId?: string): Promise<string> {
|
||
const tab = tabId ? this.tabs.get(tabId) : this.activeTabId ? this.tabs.get(this.activeTabId) : null
|
||
if (!tab) throw new Error('标签页不存在，无法截取画面。')
|
||
const image = await tab.view.webContents.capturePage()
|
||
return image.toDataURL()
|
||
}
|
||
}
|
||
|
||
const MAX_BROWSER_OBSERVE_DEPTH_FALLBACK = 400
|
||
|
||
let controllerInstance: BrowserController | null = null
|
||
|
||
/** 惰性单例：首次取用时才创建（session 相关 API 需在 app ready 后调用） */
|
||
export function getBrowserController(): BrowserController {
|
||
if (!controllerInstance) controllerInstance = new BrowserController()
|
||
return controllerInstance
|
||
}
|