feat: 内置受管浏览器(CDP + AX 快照 ref 体系)+ 本地浏览器配置导入
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* CDP 命令的基础设施:超时与中止。
|
||||
*
|
||||
* Electron Debugger 的 sendCommand 在目标页面卡死或 DevTools 通道异常时可能永不
|
||||
* settle;超时/中止只让调用方继续执行,底层命令后续 settle 时会被安全忽略。
|
||||
*/
|
||||
|
||||
export const BROWSER_CDP_COMMAND_TIMEOUT_MS = 8_000
|
||||
export const BROWSER_OBSERVE_TIMEOUT_MS = 5_000
|
||||
export const BROWSER_OBSERVE_AX_DEPTH = 8
|
||||
export const BROWSER_OBSERVE_EXTENDED_AX_DEPTH = 16
|
||||
|
||||
/** 高预算 Observe 读取更深的 AX tree,常规观察保持紧凑避免 Chromium 长时间阻塞 */
|
||||
export function resolveBrowserObserveAxDepth(maxElements: number): number {
|
||||
return maxElements > 240 ? BROWSER_OBSERVE_EXTENDED_AX_DEPTH : BROWSER_OBSERVE_AX_DEPTH
|
||||
}
|
||||
|
||||
export class BrowserCdpTimeoutError extends Error {
|
||||
constructor(method: string, timeoutMs: number) {
|
||||
super(`浏览器页面未在 ${Math.ceil(timeoutMs / 1_000)} 秒内响应 ${method},请稍后重试或重新加载页面。`)
|
||||
this.name = 'BrowserCdpTimeoutError'
|
||||
}
|
||||
}
|
||||
|
||||
/** 已发出的 DevTools 命令无法撤销;该错误只保证不再执行后续页面动作 */
|
||||
export class BrowserOperationAbortedError extends Error {
|
||||
constructor() {
|
||||
super('浏览器操作已停止。已发送的页面指令可能已执行,页面状态请重新观察确认。')
|
||||
this.name = 'BrowserOperationAbortedError'
|
||||
}
|
||||
}
|
||||
|
||||
export function throwIfBrowserOperationAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) throw new BrowserOperationAbortedError()
|
||||
}
|
||||
|
||||
export interface CdpResponse {
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
export function withBrowserCdpTimeout<T>(
|
||||
command: () => Promise<T>,
|
||||
method: string,
|
||||
timeoutMs: number = BROWSER_CDP_COMMAND_TIMEOUT_MS
|
||||
): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
let settled = false
|
||||
const settle = (callback: () => void): void => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
clearTimeout(timer)
|
||||
callback()
|
||||
}
|
||||
const timer = setTimeout(() => settle(() => reject(new BrowserCdpTimeoutError(method, timeoutMs))), timeoutMs)
|
||||
void Promise.resolve()
|
||||
.then(command)
|
||||
.then((value) => settle(() => resolve(value)))
|
||||
.catch((error: unknown) => settle(() => reject(error)))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,939 @@
|
||||
import { app, BrowserWindow, session as electronSession, WebContentsView, type Rectangle, type Session } from 'electron'
|
||||
import { existsSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import type { BrowserImportInfo, BrowserTabInfo, BrowserUiState } from '@shared/types'
|
||||
import {
|
||||
BrowserCdpTimeoutError,
|
||||
BROWSER_OBSERVE_TIMEOUT_MS,
|
||||
resolveBrowserObserveAxDepth,
|
||||
withBrowserCdpTimeout,
|
||||
type CdpResponse
|
||||
} from './browser-cdp'
|
||||
import {
|
||||
assertSafeBrowserUrl,
|
||||
browserObservationNameLimit,
|
||||
NAMELESS_ALLOWED_ROLES,
|
||||
prioritizeBrowserObservationCandidates,
|
||||
resolveBrowserDestination,
|
||||
resolveBrowserObserveMaxElements
|
||||
} from './browser-policy'
|
||||
|
||||
/**
|
||||
* 内置受管浏览器控制器。
|
||||
*
|
||||
* 架构(参考 Proma 的成熟实现,适配本应用单窗口/单聊天会话形态):
|
||||
* - 每个标签页 = 一个原生 WebContentsView,通过 contentView.addChildView 呈现;
|
||||
* 渲染层 BrowserSlot 量出占位矩形,经 IPC 同步边界(原生视图天然盖在 DOM 之上)。
|
||||
* - 页面观察/操作全部走 CDP(webContents.debugger):AX tree 快照 + ref 代际机制
|
||||
* + Input.dispatch* 真实输入事件;不在页面里执行模拟点击。
|
||||
* - 登录态隔离在 persist:aso-browser 分区;本地浏览器配置导入由 browser-profile.ts
|
||||
* 直接迁移分区目录数据(Cookies/Local Storage + 密钥移植)。
|
||||
*/
|
||||
|
||||
const PARTITION = 'persist:aso-browser'
|
||||
const NEW_TAB_URL = 'https://www.google.com/'
|
||||
const MAX_TABS = 12
|
||||
const MAX_FILL_CHARS = 10_000
|
||||
const MAX_SCRIPT_CHARS = 20_000
|
||||
const MAX_EXTRACT_CHARS = 50_000
|
||||
|
||||
interface RefEntry {
|
||||
backendNodeId: number
|
||||
generation: number
|
||||
label: string
|
||||
editable: boolean
|
||||
}
|
||||
|
||||
interface TabRecord {
|
||||
tabId: string
|
||||
view: WebContentsView
|
||||
createdAt: number
|
||||
url: string
|
||||
title: string
|
||||
loading: boolean
|
||||
/** ref 表与代际号:每次观察/导航后旧 ref 失效 */
|
||||
refs: Map<string, RefEntry>
|
||||
generation: number
|
||||
/** 同一 tab 的 Agent 操作串行化,避免 CDP 命令交错 */
|
||||
commandTail: Promise<unknown>
|
||||
lastLayoutRevision: number
|
||||
attached: boolean
|
||||
}
|
||||
|
||||
interface BrowserSettings {
|
||||
riskAcknowledged: boolean
|
||||
imported?: BrowserImportInfo | null
|
||||
}
|
||||
|
||||
export interface BrowserObservation {
|
||||
tabId: string
|
||||
url: string
|
||||
title: string
|
||||
generation: number
|
||||
elements: Array<{ ref: string; role: string; name: string; editable: boolean }>
|
||||
}
|
||||
|
||||
export interface BrowserWaitCondition {
|
||||
kind: 'url' | 'text' | 'selector'
|
||||
value: string
|
||||
}
|
||||
|
||||
function textValue(v: unknown): string {
|
||||
if (v && typeof v === 'object') {
|
||||
const value = (v as { value?: unknown }).value
|
||||
if (typeof value === 'string') return value
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function isEditableAxNode(node: Record<string, unknown>): boolean {
|
||||
const role = textValue(node.role).toLowerCase()
|
||||
if (role === 'textfield' || role === 'textbox' || role === 'searchbox') return true
|
||||
const props = Array.isArray(node.properties) ? (node.properties as Array<Record<string, unknown>>) : []
|
||||
for (const p of props) {
|
||||
if (String(p.name) === 'editable') {
|
||||
const inner = p.value as { value?: unknown } | undefined
|
||||
if (inner && typeof inner.value === 'boolean') return inner.value
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** 导航键 → CDP key 事件参数(rawKeyDown + VK code 才会触发默认行为:滚动/提交/焦点移动) */
|
||||
const KEY_EVENTS: Record<string, { key: string; code: string; vk: number }> = {
|
||||
enter: { key: 'Enter', code: 'Enter', vk: 13 },
|
||||
tab: { key: 'Tab', code: 'Tab', vk: 9 },
|
||||
escape: { key: 'Escape', code: 'Escape', vk: 27 },
|
||||
backspace: { key: 'Backspace', code: 'Backspace', vk: 8 },
|
||||
delete: { key: 'Delete', code: 'Delete', vk: 46 },
|
||||
pageup: { key: 'PageUp', code: 'PageUp', vk: 33 },
|
||||
pagedown: { key: 'PageDown', code: 'PageDown', vk: 34 },
|
||||
home: { key: 'Home', code: 'Home', vk: 36 },
|
||||
end: { key: 'End', code: 'End', vk: 35 },
|
||||
arrowup: { key: 'ArrowUp', code: 'ArrowUp', vk: 38 },
|
||||
arrowdown: { key: 'ArrowDown', code: 'ArrowDown', vk: 40 },
|
||||
arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft', vk: 37 },
|
||||
arrowright: { key: 'ArrowRight', code: 'ArrowRight', vk: 39 },
|
||||
space: { key: ' ', code: 'Space', vk: 32 }
|
||||
}
|
||||
|
||||
export class BrowserController {
|
||||
private owner: BrowserWindow | null = null
|
||||
private browserSession: Session
|
||||
private tabs = new Map<string, TabRecord>()
|
||||
private activeTabId: string | null = null
|
||||
private agentTabId: string | null = null
|
||||
private listeners = new Set<(state: BrowserUiState) => void>()
|
||||
private traceText: string | null = null
|
||||
private settings: BrowserSettings = { riskAcknowledged: false }
|
||||
|
||||
constructor() {
|
||||
this.browserSession = electronSession.fromPartition(PARTITION)
|
||||
// 诚实的 UA:保留 Chromium token 保证站点兼容,去掉 Electron 标识
|
||||
this.browserSession.setUserAgent(
|
||||
this.browserSession.getUserAgent().replace(/Electron\/[\d.]+/, `InkreachAgent/${app.getVersion()}`)
|
||||
)
|
||||
// 网页权限请求(摄像头/定位等)默认拒绝
|
||||
this.browserSession.setPermissionRequestHandler((_wc, _permission, callback) => callback(false))
|
||||
// 下载留在受管浏览器内,落到系统下载目录(重名自动加序号)
|
||||
this.browserSession.on('will-download', (_event, item) => {
|
||||
const dir = app.getPath('downloads')
|
||||
const base = item.getFilename() || 'download'
|
||||
const dot = base.lastIndexOf('.')
|
||||
const stem = dot > 0 ? base.slice(0, dot) : base
|
||||
const ext = dot > 0 ? base.slice(dot) : ''
|
||||
let target = join(dir, base)
|
||||
for (let i = 1; existsSync(target) && i < 100; i++) {
|
||||
target = join(dir, `${stem} (${i})${ext}`)
|
||||
}
|
||||
item.setSavePath(target)
|
||||
})
|
||||
this.settings = this.loadSettings()
|
||||
}
|
||||
|
||||
// ---- 设置持久化 ----
|
||||
|
||||
private get settingsFile(): string {
|
||||
return join(app.getPath('userData'), 'aso-browser.json')
|
||||
}
|
||||
|
||||
private loadSettings(): BrowserSettings {
|
||||
try {
|
||||
const raw = JSON.parse(readFileSync(this.settingsFile, 'utf8')) as Partial<BrowserSettings>
|
||||
return {
|
||||
riskAcknowledged: raw.riskAcknowledged === true,
|
||||
imported: raw.imported ?? null
|
||||
}
|
||||
} catch {
|
||||
return { riskAcknowledged: false }
|
||||
}
|
||||
}
|
||||
|
||||
private saveSettings(): void {
|
||||
try {
|
||||
writeFileSync(this.settingsFile, JSON.stringify(this.settings, null, 2), 'utf8')
|
||||
} catch (err) {
|
||||
console.error('[browser] save settings failed:', err)
|
||||
}
|
||||
}
|
||||
|
||||
/** 配置导入完成后由 browser-profile.ts 回写摘要 */
|
||||
setImportedInfo(info: BrowserImportInfo | null): void {
|
||||
this.settings.imported = info
|
||||
this.saveSettings()
|
||||
this.emitState()
|
||||
}
|
||||
|
||||
get importedInfo(): BrowserImportInfo | null {
|
||||
return this.settings.imported ?? null
|
||||
}
|
||||
|
||||
/** 当前是否有活跃标签页(导入配置前必须没有) */
|
||||
get hasTabs(): boolean {
|
||||
return this.tabs.size > 0
|
||||
}
|
||||
|
||||
// ---- 窗口与状态广播 ----
|
||||
|
||||
setOwnerWindow(win: BrowserWindow): void {
|
||||
this.owner = win
|
||||
}
|
||||
|
||||
handleWindowClosed(): void {
|
||||
for (const tab of this.tabs.values()) {
|
||||
try {
|
||||
if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach()
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
try {
|
||||
tab.view.setVisible(false)
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
}
|
||||
this.tabs.clear()
|
||||
this.activeTabId = null
|
||||
this.agentTabId = null
|
||||
this.owner = null
|
||||
}
|
||||
|
||||
onState(fn: (state: BrowserUiState) => void): () => void {
|
||||
this.listeners.add(fn)
|
||||
return () => this.listeners.delete(fn)
|
||||
}
|
||||
|
||||
private buildState(): BrowserUiState {
|
||||
const tabs: BrowserTabInfo[] = [...this.tabs.values()]
|
||||
.sort((a, b) => a.createdAt - b.createdAt)
|
||||
.map((t) => ({ tabId: t.tabId, url: t.url, title: t.title, loading: t.loading }))
|
||||
return {
|
||||
open: tabs.length > 0,
|
||||
riskAcknowledged: this.settings.riskAcknowledged,
|
||||
tabs,
|
||||
activeTabId: this.activeTabId,
|
||||
trace: this.traceText,
|
||||
imported: this.settings.imported ?? null
|
||||
}
|
||||
}
|
||||
|
||||
getState(): BrowserUiState {
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
private emitState(): void {
|
||||
const state = this.buildState()
|
||||
for (const fn of this.listeners) {
|
||||
try {
|
||||
fn(state)
|
||||
} catch (err) {
|
||||
console.error('[browser] state listener error:', err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private trace(tab: TabRecord, action: string, detail: string): void {
|
||||
this.traceText = `[${action}] ${detail}`
|
||||
void tab
|
||||
this.emitState()
|
||||
}
|
||||
|
||||
// ---- 风险告知门禁 ----
|
||||
|
||||
acknowledgeRisk(): void {
|
||||
this.settings.riskAcknowledged = true
|
||||
this.saveSettings()
|
||||
this.emitState()
|
||||
}
|
||||
|
||||
private assertRiskAcknowledged(): void {
|
||||
if (!this.settings.riskAcknowledged) {
|
||||
throw new Error('内置浏览器首次使用尚未确认风险告知:请在应用右侧浏览器面板中阅读并点击「我已了解,开始使用」后重试。')
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 会话与标签页 ----
|
||||
|
||||
private ensureBrowser(): TabRecord {
|
||||
if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪,无法使用内置浏览器。')
|
||||
if (this.tabs.size === 0) this.createTab(NEW_TAB_URL)
|
||||
const active = this.activeTabId ? this.tabs.get(this.activeTabId) : null
|
||||
const fallback = this.tabs.values().next().value
|
||||
const tab = active ?? fallback
|
||||
if (!tab) throw new Error('浏览器标签页不可用。')
|
||||
this.emitState()
|
||||
return tab
|
||||
}
|
||||
|
||||
private invalidateTabDocument(tab: TabRecord): void {
|
||||
tab.generation += 1
|
||||
tab.refs.clear()
|
||||
}
|
||||
|
||||
private createTab(initialUrl: string, viewOptions?: Electron.WebContentsViewConstructorOptions): TabRecord {
|
||||
if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪,无法创建浏览器标签。')
|
||||
if (this.tabs.size >= MAX_TABS) throw new Error(`浏览器标签页已达上限(${MAX_TABS}),请先关闭不需要的标签。`)
|
||||
const tabId = `tab-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
||||
const view = new WebContentsView(
|
||||
viewOptions ?? {
|
||||
webPreferences: {
|
||||
partition: PARTITION,
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
webSecurity: true
|
||||
}
|
||||
}
|
||||
)
|
||||
const tab: TabRecord = {
|
||||
tabId,
|
||||
view,
|
||||
createdAt: Date.now(),
|
||||
url: '',
|
||||
title: '新标签页',
|
||||
loading: false,
|
||||
refs: new Map(),
|
||||
generation: 0,
|
||||
commandTail: Promise.resolve(),
|
||||
lastLayoutRevision: 0,
|
||||
attached: false
|
||||
}
|
||||
view.setVisible(false)
|
||||
|
||||
// popup 收编为受管同级标签(必须同步返回 webContents,故提前建好 TabRecord)
|
||||
view.webContents.setWindowOpenHandler(({ url }) => {
|
||||
try {
|
||||
assertSafeBrowserUrl(url)
|
||||
} catch {
|
||||
this.trace(tab, 'popup', `已阻止不安全的新窗口: ${url.slice(0, 80)}`)
|
||||
return { action: 'deny' }
|
||||
}
|
||||
if (this.tabs.get(tabId) !== tab || this.tabs.size >= MAX_TABS) return { action: 'deny' }
|
||||
return {
|
||||
action: 'allow',
|
||||
outlivesOpener: true,
|
||||
createWindow: (options) => {
|
||||
const popup = this.createTab(url, options)
|
||||
this.selectTab(popup.tabId)
|
||||
this.trace(popup, 'popup', '网页打开了新标签页')
|
||||
return popup.view.webContents
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const updateNav = (): void => {
|
||||
const wc = view.webContents
|
||||
if (wc.isDestroyed()) return
|
||||
try {
|
||||
tab.url = wc.getURL()
|
||||
tab.title = wc.getTitle() || tab.url
|
||||
tab.loading = wc.isLoading()
|
||||
} catch {
|
||||
// 竞态时忽略
|
||||
}
|
||||
this.emitState()
|
||||
}
|
||||
view.webContents.on('did-start-loading', () => {
|
||||
tab.loading = true
|
||||
this.invalidateTabDocument(tab)
|
||||
this.emitState()
|
||||
})
|
||||
view.webContents.on('did-stop-loading', updateNav)
|
||||
view.webContents.on('page-title-updated', updateNav)
|
||||
view.webContents.on('did-navigate', () => {
|
||||
this.invalidateTabDocument(tab)
|
||||
updateNav()
|
||||
})
|
||||
view.webContents.on('did-navigate-in-page', () => {
|
||||
this.invalidateTabDocument(tab)
|
||||
updateNav()
|
||||
})
|
||||
view.webContents.on('destroyed', () => {
|
||||
this.tabs.delete(tabId)
|
||||
this.removeView(view)
|
||||
if (this.activeTabId === tabId) this.activeTabId = this.tabs.keys().next().value ?? null
|
||||
if (this.agentTabId === tabId) this.agentTabId = this.activeTabId
|
||||
this.emitState()
|
||||
})
|
||||
|
||||
try {
|
||||
view.webContents.debugger.attach('1.3')
|
||||
} catch (err) {
|
||||
console.warn('[browser] CDP attach 失败:', err)
|
||||
}
|
||||
this.tabs.set(tabId, tab)
|
||||
if (!this.activeTabId) this.activeTabId = tabId
|
||||
// popup 场景(带 viewOptions)由 Electron 的 createWindow 机制自行导航,避免双重加载
|
||||
if (initialUrl && !viewOptions) void view.webContents.loadURL(assertSafeBrowserUrl(initialUrl)).catch(() => undefined)
|
||||
return tab
|
||||
}
|
||||
|
||||
private removeView(view: WebContentsView): void {
|
||||
try {
|
||||
this.owner?.contentView.removeChildView(view)
|
||||
} catch {
|
||||
// 窗口已销毁
|
||||
}
|
||||
try {
|
||||
view.webContents.close()
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
}
|
||||
|
||||
listTabs(): BrowserUiState {
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
newTab(url?: string, byAgent = true): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.createTab(url ? resolveBrowserDestination(url) : NEW_TAB_URL)
|
||||
this.activeTabId = tab.tabId
|
||||
if (byAgent) this.agentTabId = tab.tabId
|
||||
this.emitState()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
selectTab(tabId: string): BrowserUiState {
|
||||
const tab = this.tabs.get(tabId)
|
||||
if (!tab) throw new Error(`标签页不存在: ${tabId}`)
|
||||
this.activeTabId = tabId
|
||||
try {
|
||||
tab.view.webContents.focus()
|
||||
} catch {
|
||||
// 页面可能正在加载
|
||||
}
|
||||
this.emitState()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
closeTab(tabId?: string): BrowserUiState {
|
||||
const id = tabId ?? this.activeTabId
|
||||
if (!id) return this.buildState()
|
||||
const tab = this.tabs.get(id)
|
||||
if (!tab) return this.buildState()
|
||||
try {
|
||||
if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach()
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
this.removeView(tab.view)
|
||||
this.tabs.delete(id)
|
||||
if (this.activeTabId === id) this.activeTabId = this.tabs.keys().next().value ?? null
|
||||
if (this.agentTabId === id) this.agentTabId = this.activeTabId
|
||||
this.emitState()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
/** 关闭整个受管浏览器(销毁全部标签;导入配置前也会走这里) */
|
||||
close(): BrowserUiState {
|
||||
for (const tabId of [...this.tabs.keys()]) this.closeTab(tabId)
|
||||
this.agentTabId = null
|
||||
this.traceText = null
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
// ---- 渲染层布局同步 ----
|
||||
|
||||
applyLayout(layout: { tabId: string; visible: boolean; bounds: Rectangle; revision: number }): void {
|
||||
const tab = this.tabs.get(layout.tabId)
|
||||
if (!tab || !this.owner || this.owner.isDestroyed()) return
|
||||
if (layout.revision <= tab.lastLayoutRevision) return
|
||||
tab.lastLayoutRevision = layout.revision
|
||||
try {
|
||||
if (layout.visible && layout.bounds.width > 4 && layout.bounds.height > 4) {
|
||||
if (!tab.attached) {
|
||||
this.owner.contentView.addChildView(tab.view)
|
||||
tab.attached = true
|
||||
}
|
||||
tab.view.setBounds(layout.bounds)
|
||||
tab.view.setVisible(true)
|
||||
} else {
|
||||
tab.view.setVisible(false)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[browser] apply layout failed:', err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 用户导航(地址栏/按钮) ----
|
||||
|
||||
navigateDisplay(input: string): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
void tab.view.webContents.loadURL(resolveBrowserDestination(input)).catch(() => undefined)
|
||||
this.trace(tab, 'navigate', `打开 ${input.slice(0, 80)}`)
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
goBack(): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
if (tab.view.webContents.navigationHistory.canGoBack()) tab.view.webContents.navigationHistory.goBack()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
goForward(): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
if (tab.view.webContents.navigationHistory.canGoForward()) tab.view.webContents.navigationHistory.goForward()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
reload(): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
tab.view.webContents.reload()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
// ---- CDP 基础 ----
|
||||
|
||||
private cdp(tab: TabRecord, method: string, params?: Record<string, unknown>, timeoutMs?: number): Promise<CdpResponse> {
|
||||
const debuggerClient = tab.view.webContents.debugger
|
||||
if (!debuggerClient.isAttached()) throw new Error('浏览器调试通道不可用,请重新打开该标签页。')
|
||||
return withBrowserCdpTimeout(() => debuggerClient.sendCommand(method, params) as Promise<CdpResponse>, method, timeoutMs).catch(
|
||||
(error: unknown) => {
|
||||
if (error instanceof BrowserCdpTimeoutError) this.recoverDebugger(tab, method)
|
||||
throw error
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/** 超时后重连 debugger,避免一个卡死页面让后续浏览器调用全部失败 */
|
||||
private recoverDebugger(tab: TabRecord, method: string): void {
|
||||
try {
|
||||
const debuggerClient = tab.view.webContents.debugger
|
||||
if (debuggerClient.isAttached()) debuggerClient.detach()
|
||||
debuggerClient.attach('1.3')
|
||||
console.warn(`[browser] ${method} 超时,已重连调试通道`)
|
||||
} catch {
|
||||
// 页面已关闭
|
||||
}
|
||||
}
|
||||
|
||||
/** Agent 的 tab 内操作串行化(用户操作不排队) */
|
||||
private runAgentOp<T>(tab: TabRecord, fn: () => Promise<T>): Promise<T> {
|
||||
const next = tab.commandTail.then(fn, fn)
|
||||
tab.commandTail = next.catch(() => undefined)
|
||||
return next
|
||||
}
|
||||
|
||||
private getAgentTab(tabId?: string): TabRecord {
|
||||
if (tabId) {
|
||||
const explicit = this.tabs.get(tabId)
|
||||
if (!explicit) throw new Error('目标标签页不存在,请先 browser_list_tabs 并用 browser_new_tab/browser_select_tab 选择。')
|
||||
return explicit
|
||||
}
|
||||
const id = this.agentTabId ?? this.activeTabId
|
||||
const tab = id ? this.tabs.get(id) : null
|
||||
if (tab) return tab
|
||||
// Agent 首次操作时还没有任何标签页:自动开一个工作页
|
||||
return this.ensureBrowser()
|
||||
}
|
||||
|
||||
private assertCurrentDocument(tab: TabRecord, generation: number): void {
|
||||
if (tab.generation !== generation) {
|
||||
throw new Error('页面已变化(导航或重渲染),旧元素引用失效:请重新 browser_observe 后再操作。')
|
||||
}
|
||||
}
|
||||
|
||||
private resolveRef(tab: TabRecord, ref: string): RefEntry {
|
||||
const entry = tab.refs.get(ref)
|
||||
if (!entry || entry.generation !== tab.generation) {
|
||||
throw new Error('元素引用已失效,请先重新调用 browser_observe。')
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
/** AX ref → 视口中心坐标(先滚动到可见再读 box;懒加载列表滚动后坐标会变,不能复用旧值) */
|
||||
private async centerForRef(tab: TabRecord, ref: string, generation: number): Promise<{ x: number; y: number }> {
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
const { backendNodeId } = this.resolveRef(tab, ref)
|
||||
await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
const box = await this.cdp(tab, 'DOM.getBoxModel', { backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
const model = box.model as Record<string, unknown> | undefined
|
||||
const quad = Array.isArray(model?.content) ? (model?.content as unknown[]) : []
|
||||
if (quad.length < 8 || !quad.every((v) => typeof v === 'number')) {
|
||||
throw new Error('目标元素当前不可点击,请重新观察页面。')
|
||||
}
|
||||
const n = quad as number[]
|
||||
return {
|
||||
x: (n[0] + n[2] + n[4] + n[6]) / 4,
|
||||
y: (n[1] + n[3] + n[5] + n[7]) / 4
|
||||
}
|
||||
}
|
||||
|
||||
/** 点击前在页面上短暂高亮目标,用户能看清 Agent 正在操作什么 */
|
||||
private async highlightAgentTarget(tab: TabRecord, backendNodeId: number): Promise<void> {
|
||||
try {
|
||||
const resolved = await this.cdp(tab, 'DOM.resolveNode', { backendNodeId })
|
||||
const objectId = (resolved.object as { objectId?: string } | undefined)?.objectId
|
||||
if (!objectId) return
|
||||
await this.cdp(tab, 'Runtime.callFunctionOn', {
|
||||
functionDeclaration:
|
||||
'function(){const r=this.getBoundingClientRect();const d=document.createElement("div");' +
|
||||
'd.style.cssText=`position:fixed;left:${r.left-2}px;top:${r.top-2}px;width:${r.width+4}px;height:${r.height+4}px;`+' +
|
||||
'"outline:2px solid #f59e0b;outline-offset:1px;pointer-events:none;z-index:2147483647;";' +
|
||||
'document.documentElement.appendChild(d);setTimeout(()=>d.remove(),700);return true}',
|
||||
objectId,
|
||||
returnByValue: true
|
||||
})
|
||||
} catch {
|
||||
// 高亮失败不影响操作
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Agent 工具实现 ----
|
||||
|
||||
async navigate(url: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
this.agentTabId = tab.tabId
|
||||
this.activeTabId = tab.tabId
|
||||
const destination = resolveBrowserDestination(url)
|
||||
await tab.view.webContents.loadURL(destination)
|
||||
this.trace(tab, 'navigate', `打开 ${destination.slice(0, 80)}`)
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
private parseAxNodes(response: CdpResponse): Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> {
|
||||
const nodes = Array.isArray(response.nodes) ? (response.nodes as Array<Record<string, unknown>>) : []
|
||||
const candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> = []
|
||||
for (const node of nodes) {
|
||||
if (!node) continue
|
||||
const backendNodeId = typeof node.backendDOMNodeId === 'number' ? node.backendDOMNodeId : 0
|
||||
const role = textValue(node.role).toLowerCase()
|
||||
const name = textValue(node.name)
|
||||
const editable = isEditableAxNode(node)
|
||||
if (!backendNodeId || !role) continue
|
||||
if (!name && !editable && !NAMELESS_ALLOWED_ROLES.has(role)) continue
|
||||
candidates.push({ backendNodeId, role, name: name.slice(0, browserObservationNameLimit(role)), editable })
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
private buildRefs(tab: TabRecord, candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }>): BrowserObservation['elements'] {
|
||||
tab.generation += 1
|
||||
tab.refs.clear()
|
||||
const elements: BrowserObservation['elements'] = []
|
||||
for (const candidate of candidates) {
|
||||
const ref = `r${tab.generation}-${elements.length + 1}`
|
||||
tab.refs.set(ref, {
|
||||
backendNodeId: candidate.backendNodeId,
|
||||
generation: tab.generation,
|
||||
label: candidate.name ? `${candidate.role}「${candidate.name.slice(0, 80)}」` : candidate.role,
|
||||
editable: candidate.editable
|
||||
})
|
||||
elements.push({ ref, role: candidate.role, name: candidate.name, editable: candidate.editable })
|
||||
}
|
||||
return elements
|
||||
}
|
||||
|
||||
async observe(tabId?: string, requestedMaxElements?: number): Promise<BrowserObservation> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
this.agentTabId = tab.tabId
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const maxElements = resolveBrowserObserveMaxElements(requestedMaxElements)
|
||||
const depth = resolveBrowserObserveAxDepth(maxElements)
|
||||
const response = await this.cdp(
|
||||
tab,
|
||||
'Accessibility.getFullAXTree',
|
||||
{ depth },
|
||||
BROWSER_OBSERVE_TIMEOUT_MS
|
||||
)
|
||||
const selected = prioritizeBrowserObservationCandidates(this.parseAxNodes(response), maxElements)
|
||||
const elements = this.buildRefs(tab, selected)
|
||||
this.trace(tab, 'observe', `读取到 ${elements.length}/${maxElements} 个元素(AX 深度 ${depth})`)
|
||||
return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements }
|
||||
})
|
||||
}
|
||||
|
||||
async find(
|
||||
query: { role?: string; name?: string; exact?: boolean; maxResults?: number },
|
||||
tabId?: string
|
||||
): Promise<BrowserObservation> {
|
||||
const role = query.role?.trim().toLowerCase()
|
||||
const name = query.name?.trim().toLowerCase()
|
||||
const maxResults = query.maxResults ?? 20
|
||||
if (!role && !name) throw new Error('语义定位至少需要 role 或 name。')
|
||||
if (!Number.isInteger(maxResults) || maxResults < 1 || maxResults > 50) throw new Error('maxResults 必须是 1 到 50 的整数。')
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
this.agentTabId = tab.tabId
|
||||
return this.runAgentOp(tab, async () => {
|
||||
// 过滤后再套观察配额:Find 能找到被紧凑 Observe 截掉的目标
|
||||
const response = await this.cdp(
|
||||
tab,
|
||||
'Accessibility.getFullAXTree',
|
||||
{ depth: resolveBrowserObserveAxDepth(MAX_BROWSER_OBSERVE_DEPTH_FALLBACK) },
|
||||
BROWSER_OBSERVE_TIMEOUT_MS
|
||||
)
|
||||
const candidates = this.parseAxNodes(response).filter((candidate) => {
|
||||
const roleMatches = !role || candidate.role === role
|
||||
const candidateName = candidate.name.toLowerCase()
|
||||
const nameMatches = !name || (query.exact ? candidateName === name : candidateName.includes(name))
|
||||
return roleMatches && nameMatches
|
||||
})
|
||||
const elements = this.buildRefs(tab, candidates.slice(0, maxResults))
|
||||
this.trace(tab, 'find', `语义定位到 ${elements.length} 个元素(${role ?? name})`)
|
||||
return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements }
|
||||
})
|
||||
}
|
||||
|
||||
private async clickRef(tab: TabRecord, ref: string, generation: number): Promise<RefEntry> {
|
||||
const target = this.resolveRef(tab, ref)
|
||||
const { x, y } = await this.centerForRef(tab, ref, generation)
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 })
|
||||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 })
|
||||
return target
|
||||
}
|
||||
|
||||
async click(ref: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = await this.clickRef(tab, ref, generation)
|
||||
this.trace(tab, 'click', `点击 ${target.label}`)
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
/** 点击 + 等待合并为一次串行操作(已知点击后预期状态时优先使用) */
|
||||
async act(ref: string, waitFor: BrowserWaitCondition | undefined, timeoutMs = 10_000, tabId?: string): Promise<{ state: BrowserUiState; matched: boolean }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = await this.clickRef(tab, ref, generation)
|
||||
this.trace(tab, 'click', `点击 ${target.label}`)
|
||||
if (!waitFor) return { state: this.buildState(), matched: true }
|
||||
const matched = await this.waitForInternal(tab, waitFor, timeoutMs)
|
||||
return { state: this.buildState(), matched }
|
||||
})
|
||||
}
|
||||
|
||||
async hover(ref: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = this.resolveRef(tab, ref)
|
||||
const { x, y } = await this.centerForRef(tab, ref, generation)
|
||||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y })
|
||||
this.trace(tab, 'hover', `悬停 ${target.label}`)
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
async fill(ref: string, text: string, tabId?: string): Promise<BrowserUiState> {
|
||||
if (text.length > MAX_FILL_CHARS) throw new Error(`单次输入不能超过 ${MAX_FILL_CHARS} 个字符。`)
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = this.resolveRef(tab, ref)
|
||||
if (!target.editable) throw new Error('目标元素不是可编辑字段,请重新观察后选择 input、textarea 或 contenteditable。')
|
||||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||||
await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.cdp(tab, 'DOM.focus', { backendNodeId: target.backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
// 全选(Ctrl/Cmd+A)后 insertText 整段替换,不逐字符模拟
|
||||
const selectAllModifier = process.platform === 'darwin' ? 4 : 2
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', {
|
||||
type: 'keyDown',
|
||||
key: 'a',
|
||||
code: 'KeyA',
|
||||
windowsVirtualKeyCode: 65,
|
||||
modifiers: selectAllModifier
|
||||
})
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', {
|
||||
type: 'keyUp',
|
||||
key: 'a',
|
||||
code: 'KeyA',
|
||||
windowsVirtualKeyCode: 65,
|
||||
modifiers: selectAllModifier
|
||||
})
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.cdp(tab, 'Input.insertText', { text })
|
||||
this.trace(tab, 'fill', `在 ${target.label} 输入 ${Array.from(text).length} 个字符`)
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
async press(key: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const normalized = key.trim().toLowerCase().replace(/\s+/g, '')
|
||||
const keyEvent = KEY_EVENTS[normalized]
|
||||
if (keyEvent) {
|
||||
const event = { key: keyEvent.key, code: keyEvent.code, windowsVirtualKeyCode: keyEvent.vk }
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'rawKeyDown', ...event })
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'keyUp', ...event })
|
||||
this.trace(tab, 'press', `按下 ${keyEvent.key}`)
|
||||
} else {
|
||||
await this.cdp(tab, 'Input.insertText', { text: key })
|
||||
this.trace(tab, 'press', `输入 ${Array.from(key).length} 个字符`)
|
||||
}
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
async scroll(
|
||||
input: { deltaY?: number; position?: 'top' | 'bottom'; selector?: string },
|
||||
tabId?: string
|
||||
): Promise<{ before: { top: number; height: number }; after: { top: number; height: number }; moved: boolean }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : ''
|
||||
const position = input.position === 'top' || input.position === 'bottom' ? input.position : undefined
|
||||
const deltaY = Math.max(-50_000, Math.min(50_000, Math.round(input.deltaY ?? 600)))
|
||||
// 固定表达式:数据由 JSON 注入,不拼接用户脚本
|
||||
const expression =
|
||||
'((input)=>{const el=input.selector?document.querySelector(input.selector):(document.scrollingElement||document.documentElement);' +
|
||||
'if(!el)return {ok:false};const read=()=>({top:Math.round(el.scrollTop),height:Math.round(el.scrollHeight)});' +
|
||||
'const before=read();const top=input.position==="top"?0:input.position==="bottom"?Math.max(0,el.scrollHeight-el.clientHeight):el.scrollTop+input.deltaY;' +
|
||||
'el.scrollTo({top});const after=read();return {ok:true,before,after}})(' +
|
||||
JSON.stringify({ selector, position, deltaY }) +
|
||||
')'
|
||||
const response = await this.evaluateOnPage(tab, expression)
|
||||
const result = (response ?? {}) as { ok?: boolean; before?: { top: number; height: number }; after?: { top: number; height: number } }
|
||||
if (!result.ok || !result.before || !result.after) throw new Error('滚动目标不存在或不支持滚动,请检查 selector。')
|
||||
this.trace(tab, 'scroll', `滚动页面${selector ? `(${selector.slice(0, 40)})` : ''}`)
|
||||
return { before: result.before, after: result.after, moved: result.after.top !== result.before.top }
|
||||
})
|
||||
}
|
||||
|
||||
async extract(
|
||||
input: { selector?: string; maxChars?: number },
|
||||
tabId?: string
|
||||
): Promise<{ url: string; title: string; text: string; truncated: boolean }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : ''
|
||||
const maxChars = Math.min(MAX_EXTRACT_CHARS, Math.max(500, Math.round(input.maxChars ?? MAX_EXTRACT_CHARS)))
|
||||
const expression =
|
||||
'((input)=>{const root=input.selector?document.querySelector(input.selector):document.body;' +
|
||||
'if(!root)return null;const text=(root.innerText||"").replace(/\\n{3,}/g,"\\n\\n");' +
|
||||
'return {text:text.slice(0,input.maxChars),truncated:text.length>input.maxChars}})(' +
|
||||
JSON.stringify({ selector, maxChars }) +
|
||||
')'
|
||||
const response = await this.evaluateOnPage(tab, expression)
|
||||
const result = (response ?? {}) as { text?: string; truncated?: boolean } | null
|
||||
if (!result || typeof result.text !== 'string') throw new Error('提取目标不存在,请检查 selector。')
|
||||
this.trace(tab, 'extract', `提取正文 ${result.text.length} 字符${selector ? `(${selector.slice(0, 40)})` : ''}`)
|
||||
return { url: tab.url, title: tab.title, text: result.text, truncated: result.truncated === true }
|
||||
})
|
||||
}
|
||||
|
||||
async waitFor(condition: BrowserWaitCondition, timeoutMs = 10_000, tabId?: string): Promise<{ matched: boolean; url: string; title: string }> {
|
||||
if (!condition.value.trim()) throw new Error('等待条件不能为空。')
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs < 250 || timeoutMs > 30_000) throw new Error('等待超时必须在 250 到 30000 毫秒之间。')
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const matched = await this.waitForInternal(tab, condition, timeoutMs)
|
||||
this.trace(tab, 'wait', `${matched ? '已满足' : '超时未满足'}${condition.kind} 条件(${condition.value.slice(0, 40)})`)
|
||||
return { matched, url: tab.url, title: tab.title }
|
||||
})
|
||||
}
|
||||
|
||||
/** 轮询固定表达式(只读检查,不执行任意 JS) */
|
||||
private async waitForInternal(tab: TabRecord, condition: BrowserWaitCondition, timeoutMs: number): Promise<boolean> {
|
||||
const startedAt = Date.now()
|
||||
const payload = JSON.stringify(condition).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029')
|
||||
const expression =
|
||||
`(() => { const condition = ${payload}; try { ` +
|
||||
"if (condition.kind === 'url') return location.href.includes(condition.value); " +
|
||||
"if (condition.kind === 'text') return (document.body?.innerText || '').includes(condition.value); " +
|
||||
'return !!document.querySelector(condition.value); } catch { return false; } })()'
|
||||
while (Date.now() - startedAt <= timeoutMs) {
|
||||
const result = await this.evaluateOnPage(tab, expression)
|
||||
if (result === true) return true
|
||||
await new Promise((resolve) => setTimeout(resolve, 250))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** Agent 逃生舱:在页面上下文执行自写 JS(有长度上限,结果 JSON 化) */
|
||||
async evaluate(script: string, tabId?: string): Promise<unknown> {
|
||||
if (!script.trim()) throw new Error('脚本不能为空。')
|
||||
if (script.length > MAX_SCRIPT_CHARS) throw new Error(`脚本不能超过 ${MAX_SCRIPT_CHARS} 字符。`)
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const result = await this.evaluateOnPage(tab, script, true)
|
||||
this.trace(tab, 'js', `执行页面脚本 ${script.length} 字符`)
|
||||
return result
|
||||
})
|
||||
}
|
||||
|
||||
private async evaluateOnPage(tab: TabRecord, expression: string, userGesture = false): Promise<unknown> {
|
||||
const response = await this.cdp(tab, 'Runtime.evaluate', {
|
||||
expression,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
userGesture
|
||||
})
|
||||
const exception = response.exceptionDetails as { text?: string; exception?: { description?: string } } | undefined
|
||||
if (exception) {
|
||||
throw new Error(`页面脚本执行出错: ${exception.exception?.description || exception.text || '未知错误'}`)
|
||||
}
|
||||
const result = response.result as { type?: string; value?: unknown; description?: string } | undefined
|
||||
if (!result) return null
|
||||
if ('value' in result) return result.value ?? null
|
||||
return { type: result.type ?? 'unknown', description: result.description ?? null }
|
||||
}
|
||||
|
||||
async screenshot(tabId?: string): Promise<{ tabId: string; url: string; title: string; dataUrl: string }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const image = await tab.view.webContents.capturePage()
|
||||
this.trace(tab, 'screenshot', '截取当前页面')
|
||||
return { tabId: tab.tabId, url: tab.url, title: tab.title, dataUrl: image.toDataURL() }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_BROWSER_OBSERVE_DEPTH_FALLBACK = 400
|
||||
|
||||
let controllerInstance: BrowserController | null = null
|
||||
|
||||
/** 惰性单例:首次取用时才创建(session 相关 API 需在 app ready 后调用) */
|
||||
export function getBrowserController(): BrowserController {
|
||||
if (!controllerInstance) controllerInstance = new BrowserController()
|
||||
return controllerInstance
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* 受管浏览器的 URL 与观察策略。
|
||||
* 独立成模块便于单测与复用(渲染层不需要,主进程专用)。
|
||||
*/
|
||||
|
||||
const SEARCH_URL = 'https://www.google.com/search?q='
|
||||
|
||||
/** 导航白名单:普通 web、本地开发服务与 about:blank;file:/javascript:/data: 一律拒绝 */
|
||||
export function assertSafeBrowserUrl(input: string): string {
|
||||
const value = input.trim()
|
||||
if (!value) throw new Error('URL 不能为空。')
|
||||
if (value === 'about:blank') return value
|
||||
let parsed: URL
|
||||
try {
|
||||
parsed = new URL(value)
|
||||
} catch {
|
||||
throw new Error(`无法解析的 URL: ${value}`)
|
||||
}
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
||||
throw new Error(`不支持的地址协议(仅允许 http/https 与 about:blank): ${parsed.protocol}`)
|
||||
}
|
||||
return parsed.toString()
|
||||
}
|
||||
|
||||
function isLikelyUrl(value: string): boolean {
|
||||
const candidate = `http://${value}`
|
||||
try {
|
||||
const url = new URL(candidate)
|
||||
const host = url.hostname.toLowerCase()
|
||||
// 域名 / IP(v4/v6)/ localhost 与 *.localhost 视为直达地址
|
||||
const isHost =
|
||||
host === 'localhost' ||
|
||||
host.endsWith('.localhost') ||
|
||||
host.includes('.') ||
|
||||
host.includes(':')
|
||||
// 无空格且形如域名/IP/localhost
|
||||
return isHost && !/\s/.test(value)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** 输入归一化:URL/域名/IP/localhost 直达,普通文本走搜索 */
|
||||
export function resolveBrowserDestination(input: string): string {
|
||||
const value = input.trim()
|
||||
if (!value) throw new Error('地址不能为空。')
|
||||
if (value === 'about:blank') return value
|
||||
if (/^https?:\/\//i.test(value)) return assertSafeBrowserUrl(value)
|
||||
if (isLikelyUrl(value)) return assertSafeBrowserUrl(`https://${value}`)
|
||||
return SEARCH_URL + encodeURIComponent(value)
|
||||
}
|
||||
|
||||
// ---- Observe 观察策略 ----
|
||||
|
||||
export const DEFAULT_BROWSER_OBSERVE_MAX_ELEMENTS = 240
|
||||
export const MAX_BROWSER_OBSERVE_MAX_ELEMENTS = 400
|
||||
export const MIN_BROWSER_OBSERVE_MAX_ELEMENTS = 20
|
||||
|
||||
/** 可交互 AX role 白名单(占观察结果前 2/3 配额) */
|
||||
export const INTERACTIVE_AX_ROLES = new Set([
|
||||
'button',
|
||||
'link',
|
||||
'textfield',
|
||||
'textbox',
|
||||
'searchbox',
|
||||
'checkbox',
|
||||
'radiobutton',
|
||||
'combobox',
|
||||
'menuitem',
|
||||
'menuitemcheckbox',
|
||||
'menuitemradio',
|
||||
'tab',
|
||||
'option',
|
||||
'switch',
|
||||
'slider'
|
||||
])
|
||||
|
||||
/** 无名称也值得返回的 role(依赖 role 本身可定位) */
|
||||
export const NAMELESS_ALLOWED_ROLES = new Set(['button', 'textfield', 'textbox', 'link', 'checkbox', 'combobox'])
|
||||
|
||||
export function resolveBrowserObserveMaxElements(requested?: number): number {
|
||||
if (typeof requested !== 'number' || !Number.isFinite(requested)) return DEFAULT_BROWSER_OBSERVE_MAX_ELEMENTS
|
||||
return Math.min(MAX_BROWSER_OBSERVE_MAX_ELEMENTS, Math.max(MIN_BROWSER_OBSERVE_MAX_ELEMENTS, Math.round(requested)))
|
||||
}
|
||||
|
||||
export function isInteractiveAxRole(role: string): boolean {
|
||||
return INTERACTIVE_AX_ROLES.has(role.toLowerCase())
|
||||
}
|
||||
|
||||
export interface BrowserObservationCandidate {
|
||||
role: string
|
||||
name: string
|
||||
editable?: boolean
|
||||
}
|
||||
|
||||
/** 可交互元素优先(约 2/3 配额),其余语义上下文补足 */
|
||||
export function prioritizeBrowserObservationCandidates<T extends BrowserObservationCandidate>(
|
||||
candidates: readonly T[],
|
||||
maxElements: number
|
||||
): T[] {
|
||||
const interactiveLimit = Math.ceil(maxElements * (2 / 3))
|
||||
const interactive: T[] = []
|
||||
const context: T[] = []
|
||||
for (const candidate of candidates) {
|
||||
if (interactive.length < interactiveLimit && (isInteractiveAxRole(candidate.role) || candidate.editable)) {
|
||||
interactive.push(candidate)
|
||||
} else {
|
||||
context.push(candidate)
|
||||
}
|
||||
}
|
||||
const contextLimit = maxElements - interactive.length
|
||||
return [...interactive, ...context.slice(0, Math.max(0, contextLimit))]
|
||||
}
|
||||
|
||||
/** 不同 role 的可访问名称截断上限,避免长文本撑爆快照 */
|
||||
export function browserObservationNameLimit(role: string): number {
|
||||
if (role === 'link' || role === 'button' || role === 'menuitem') return 80
|
||||
if (role === 'heading') return 120
|
||||
return 160
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
import { app } from 'electron'
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
||||
import { homedir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import type { BrowserImportResult, BrowserProfileMeta, BrowserSource } from '@shared/types'
|
||||
import { getBrowserController } from './browser-controller'
|
||||
|
||||
/**
|
||||
* 本地浏览器配置导入(复用各网站登录态)。
|
||||
*
|
||||
* 实现思路(零解密依赖,纯文件迁移):
|
||||
* - Local Storage / IndexedDB 在 Chromium 里是明文 leveldb,直接拷入受管分区即可生效。
|
||||
* - Cookies 用 AES-GCM 加密,密钥经 OS 加密后存于浏览器根目录 Local State
|
||||
* (os_crypt.encrypted_key)。Windows 上该密钥由 DPAPI(用户级)包裹——同一
|
||||
* Windows 用户的任意进程都能解开,因此把这份密钥「移植」进本应用的 Local State,
|
||||
* 本应用内的 Chromium 即可解密拷贝过来的 Cookies(Chrome 127+ 的 app-bound
|
||||
* 加密 V20 除外,会明确提示不支持)。macOS/Linux 的密钥绑定钥匙串/KWallet,
|
||||
* Cookies 无法直接迁移,仅迁移 Local Storage。
|
||||
*/
|
||||
|
||||
/** Chromium 系浏览器的用户数据根目录(按平台) */
|
||||
const SOURCE_ROOTS: Array<{ id: string; name: string; win: string; mac: string; linux: string }> = [
|
||||
{
|
||||
id: 'chrome',
|
||||
name: 'Google Chrome',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'Google', 'Chrome', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'Google', 'Chrome'),
|
||||
linux: join(homedir(), '.config', 'google-chrome')
|
||||
},
|
||||
{
|
||||
id: 'edge',
|
||||
name: 'Microsoft Edge',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'Microsoft', 'Edge', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'Microsoft Edge'),
|
||||
linux: join(homedir(), '.config', 'microsoft-edge')
|
||||
},
|
||||
{
|
||||
id: 'brave',
|
||||
name: 'Brave',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'BraveSoftware', 'Brave-Browser', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'BraveSoftware', 'Brave-Browser'),
|
||||
linux: join(homedir(), '.config', 'BraveSoftware', 'Brave-Browser')
|
||||
},
|
||||
{
|
||||
id: 'chromium',
|
||||
name: 'Chromium',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'Chromium', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'Chromium'),
|
||||
linux: join(homedir(), '.config', 'chromium')
|
||||
}
|
||||
]
|
||||
|
||||
/** 受管浏览器分区目录(persist:aso-browser 的落盘位置) */
|
||||
function partitionDir(): string {
|
||||
return join(app.getPath('userData'), 'Partitions', 'aso-browser')
|
||||
}
|
||||
|
||||
function sourceRoot(id: string): string {
|
||||
const source = SOURCE_ROOTS.find((s) => s.id === id)
|
||||
if (!source) return ''
|
||||
return process.platform === 'win32' ? source.win : process.platform === 'darwin' ? source.mac : source.linux
|
||||
}
|
||||
|
||||
/** 读取 profile 的展示名(Preferences 的 profile.name,失败时用目录名) */
|
||||
function readProfileName(profileDir: string): string {
|
||||
try {
|
||||
const file = join(profileDir, 'Preferences')
|
||||
if (statSync(file).size > 8 * 1024 * 1024) return profileDir.split(/[\\/]/).pop() || profileDir
|
||||
const raw = JSON.parse(readFileSync(file, 'utf8')) as { profile?: { name?: string } }
|
||||
return raw.profile?.name?.trim() || profileDir.split(/[\\/]/).pop() || profileDir
|
||||
} catch {
|
||||
return profileDir.split(/[\\/]/).pop() || profileDir
|
||||
}
|
||||
}
|
||||
|
||||
/** 检测本机可导入的浏览器与 profile 列表 */
|
||||
export function detectBrowserSources(): BrowserSource[] {
|
||||
const sources: BrowserSource[] = []
|
||||
for (const source of SOURCE_ROOTS) {
|
||||
const root = sourceRoot(source.id)
|
||||
if (!root || !existsSync(root)) continue
|
||||
const profiles: BrowserProfileMeta[] = []
|
||||
try {
|
||||
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue
|
||||
if (entry.name.startsWith('.') || entry.name === 'System Profile' || entry.name === 'Guest Profile') continue
|
||||
const profileDir = join(root, entry.name)
|
||||
if (!existsSync(join(profileDir, 'Preferences'))) continue
|
||||
profiles.push({ dir: entry.name, name: readProfileName(profileDir) })
|
||||
}
|
||||
} catch {
|
||||
// 目录不可读则跳过该浏览器
|
||||
continue
|
||||
}
|
||||
if (profiles.length > 0) {
|
||||
sources.push({ id: source.id, name: source.name, userDataDir: root, profiles })
|
||||
}
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
function copyFileSafe(from: string, to: string): void {
|
||||
try {
|
||||
copyFileSync(from, to)
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code
|
||||
if (code === 'EBUSY' || code === 'EPERM' || code === 'EACCES') {
|
||||
throw new Error('源浏览器文件被占用:请完全退出对应的浏览器(含后台进程)后重试。')
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
/** 拷贝 leveldb 目录(跳过 LOCK/LOG 运行时文件),目标已存在时先清空 */
|
||||
function copyLevelDbDir(from: string, to: string): boolean {
|
||||
if (!existsSync(from)) return false
|
||||
const entries = readdirSync(from, { withFileTypes: true })
|
||||
const files = entries.filter((e) => e.isFile() && e.name !== 'LOCK' && !e.name.startsWith('LOG'))
|
||||
if (files.length === 0) return false
|
||||
rmSync(to, { recursive: true, force: true })
|
||||
mkdirSync(to, { recursive: true })
|
||||
for (const file of files) {
|
||||
copyFileSafe(join(from, file.name), join(to, file.name))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
interface SourceKeyInfo {
|
||||
key: string | null
|
||||
/** app-bound(V20)等不可移植场景的说明 */
|
||||
note?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取源浏览器 Local State 里的 os_crypt.encrypted_key 并判断可移植性。
|
||||
* Windows:前缀 DPAPI 可移植;V20 为 app-bound 加密(Chrome/Edge 127+),
|
||||
* 其他进程无法解开,明确降级。macOS/Linux:密钥在系统钥匙串/KWallet,跳过。
|
||||
*/
|
||||
function readSourceKey(userDataDir: string): SourceKeyInfo {
|
||||
if (process.platform !== 'win32') {
|
||||
return { key: null, note: '当前平台的浏览器 Cookie 密钥绑定系统钥匙串,暂不支持迁移 Cookie(LocalStorage 登录态仍会导入)。' }
|
||||
}
|
||||
try {
|
||||
const raw = JSON.parse(readFileSync(join(userDataDir, 'Local State'), 'utf8')) as {
|
||||
os_crypt?: { encrypted_key?: string }
|
||||
}
|
||||
const encoded = raw.os_crypt?.encrypted_key
|
||||
if (!encoded) return { key: null, note: '未找到源浏览器 Cookie 密钥,跳过 Cookie 迁移。' }
|
||||
const bytes = Buffer.from(encoded, 'base64')
|
||||
if (bytes.subarray(0, 3).toString('utf8') === 'V20') {
|
||||
return { key: null, note: '该浏览器使用了应用绑定加密(app-bound encryption),无法第三方迁移 Cookie;已导入 Local Storage,首次访问各站点仍需登录一次。' }
|
||||
}
|
||||
if (bytes.subarray(0, 5).toString('utf8') !== 'DPAPI') {
|
||||
return { key: null, note: '源浏览器 Cookie 密钥格式不受支持,跳过 Cookie 迁移。' }
|
||||
}
|
||||
return { key: encoded }
|
||||
} catch {
|
||||
return { key: null, note: '读取源浏览器 Local State 失败,跳过 Cookie 迁移。' }
|
||||
}
|
||||
}
|
||||
|
||||
/** 把源密钥移植进本应用根 Local State(首次会备份原文件) */
|
||||
function transplantKey(sourceKey: string): void {
|
||||
const appLocalState = join(app.getPath('userData'), 'Local State')
|
||||
const backup = `${appLocalState}.aso-backup`
|
||||
let doc: Record<string, unknown> = {}
|
||||
if (existsSync(appLocalState)) {
|
||||
try {
|
||||
doc = JSON.parse(readFileSync(appLocalState, 'utf8')) as Record<string, unknown>
|
||||
if (!existsSync(backup)) copyFileSync(appLocalState, backup)
|
||||
} catch {
|
||||
doc = {}
|
||||
}
|
||||
}
|
||||
const osCrypt = (doc.os_crypt ?? {}) as Record<string, unknown>
|
||||
osCrypt.encrypted_key = sourceKey
|
||||
doc.os_crypt = osCrypt
|
||||
writeFileSync(appLocalState, JSON.stringify(doc, null, 2), 'utf8')
|
||||
}
|
||||
|
||||
/** 导入指定浏览器的 profile 配置到受管浏览器分区 */
|
||||
export function importBrowserProfile(sourceId: string, profileDir: string): BrowserImportResult {
|
||||
const browser = getBrowserController()
|
||||
if (browser.hasTabs) {
|
||||
throw new Error('请先关闭内置浏览器的所有标签页,再导入浏览器配置。')
|
||||
}
|
||||
const root = sourceRoot(sourceId)
|
||||
if (!root || !existsSync(root)) throw new Error('未找到该浏览器的用户数据目录。')
|
||||
// 只接受该根目录下的直接子目录名,拒绝任意路径注入
|
||||
const dirName = profileDir.split(/[\\/]/).pop() ?? ''
|
||||
const sourceProfile = join(root, dirName)
|
||||
if (dirName !== profileDir || !existsSync(join(sourceProfile, 'Preferences'))) {
|
||||
throw new Error('profile 目录不存在。')
|
||||
}
|
||||
const sourceName = SOURCE_ROOTS.find((s) => s.id === sourceId)?.name ?? sourceId
|
||||
const target = partitionDir()
|
||||
|
||||
// 1) Local Storage(明文 leveldb,直接迁移)
|
||||
let localStorage = false
|
||||
try {
|
||||
localStorage = copyLevelDbDir(join(sourceProfile, 'Local Storage', 'leveldb'), join(target, 'Local Storage', 'leveldb'))
|
||||
} catch (err) {
|
||||
throw new Error(err instanceof Error ? err.message : `Local Storage 迁移失败: ${String(err)}`)
|
||||
}
|
||||
|
||||
// 2) Cookies + 密钥移植
|
||||
let cookies = false
|
||||
let cookieNote: string | undefined
|
||||
const keyInfo = readSourceKey(root)
|
||||
const cookieDb = existsSync(join(sourceProfile, 'Network', 'Cookies'))
|
||||
? join(sourceProfile, 'Network', 'Cookies')
|
||||
: existsSync(join(sourceProfile, 'Cookies'))
|
||||
? join(sourceProfile, 'Cookies')
|
||||
: null
|
||||
if (keyInfo.key && cookieDb) {
|
||||
try {
|
||||
transplantKey(keyInfo.key)
|
||||
mkdirSync(join(target, 'Network'), { recursive: true })
|
||||
copyFileSafe(cookieDb, join(target, 'Network', 'Cookies'))
|
||||
for (const suffix of ['-wal', '-shm']) {
|
||||
if (existsSync(`${cookieDb}${suffix}`)) copyFileSafe(`${cookieDb}${suffix}`, join(target, 'Network', `Cookies${suffix}`))
|
||||
}
|
||||
cookies = true
|
||||
} catch (err) {
|
||||
cookieNote = err instanceof Error ? err.message : 'Cookie 迁移失败。'
|
||||
}
|
||||
} else {
|
||||
cookieNote = keyInfo.note ?? '源浏览器没有可迁移的 Cookie 数据库。'
|
||||
}
|
||||
|
||||
const parts: string[] = []
|
||||
if (cookies) parts.push('Cookie(站点登录态)')
|
||||
if (localStorage) parts.push('LocalStorage')
|
||||
if (!cookies && !localStorage) {
|
||||
return { ok: false, cookies, localStorage, cookieNote, message: `未在 ${sourceName}(${dirName})中找到可迁移的登录数据。` }
|
||||
}
|
||||
|
||||
browser.setImportedInfo({
|
||||
browser: sourceName,
|
||||
profile: readProfileName(sourceProfile),
|
||||
at: new Date().toISOString(),
|
||||
cookies,
|
||||
localStorage,
|
||||
note: cookieNote
|
||||
})
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
cookies,
|
||||
localStorage,
|
||||
cookieNote,
|
||||
message: `已从 ${sourceName} 导入 ${parts.join(' + ')}。${cookieNote ?? ''}${
|
||||
cookies ? '若登录态未生效,重启应用后重试。' : ''
|
||||
}`
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user