feat: 内置受管浏览器(CDP + AX 快照 ref 体系)+ 本地浏览器配置导入
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* CDP 命令的基础设施:超时与中止。
|
||||
*
|
||||
* Electron Debugger 的 sendCommand 在目标页面卡死或 DevTools 通道异常时可能永不
|
||||
* settle;超时/中止只让调用方继续执行,底层命令后续 settle 时会被安全忽略。
|
||||
*/
|
||||
|
||||
export const BROWSER_CDP_COMMAND_TIMEOUT_MS = 8_000
|
||||
export const BROWSER_OBSERVE_TIMEOUT_MS = 5_000
|
||||
export const BROWSER_OBSERVE_AX_DEPTH = 8
|
||||
export const BROWSER_OBSERVE_EXTENDED_AX_DEPTH = 16
|
||||
|
||||
/** 高预算 Observe 读取更深的 AX tree,常规观察保持紧凑避免 Chromium 长时间阻塞 */
|
||||
export function resolveBrowserObserveAxDepth(maxElements: number): number {
|
||||
return maxElements > 240 ? BROWSER_OBSERVE_EXTENDED_AX_DEPTH : BROWSER_OBSERVE_AX_DEPTH
|
||||
}
|
||||
|
||||
export class BrowserCdpTimeoutError extends Error {
|
||||
constructor(method: string, timeoutMs: number) {
|
||||
super(`浏览器页面未在 ${Math.ceil(timeoutMs / 1_000)} 秒内响应 ${method},请稍后重试或重新加载页面。`)
|
||||
this.name = 'BrowserCdpTimeoutError'
|
||||
}
|
||||
}
|
||||
|
||||
/** 已发出的 DevTools 命令无法撤销;该错误只保证不再执行后续页面动作 */
|
||||
export class BrowserOperationAbortedError extends Error {
|
||||
constructor() {
|
||||
super('浏览器操作已停止。已发送的页面指令可能已执行,页面状态请重新观察确认。')
|
||||
this.name = 'BrowserOperationAbortedError'
|
||||
}
|
||||
}
|
||||
|
||||
export function throwIfBrowserOperationAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) throw new BrowserOperationAbortedError()
|
||||
}
|
||||
|
||||
export interface CdpResponse {
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
export function withBrowserCdpTimeout<T>(
|
||||
command: () => Promise<T>,
|
||||
method: string,
|
||||
timeoutMs: number = BROWSER_CDP_COMMAND_TIMEOUT_MS
|
||||
): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
let settled = false
|
||||
const settle = (callback: () => void): void => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
clearTimeout(timer)
|
||||
callback()
|
||||
}
|
||||
const timer = setTimeout(() => settle(() => reject(new BrowserCdpTimeoutError(method, timeoutMs))), timeoutMs)
|
||||
void Promise.resolve()
|
||||
.then(command)
|
||||
.then((value) => settle(() => resolve(value)))
|
||||
.catch((error: unknown) => settle(() => reject(error)))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,939 @@
|
||||
import { app, BrowserWindow, session as electronSession, WebContentsView, type Rectangle, type Session } from 'electron'
|
||||
import { existsSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import type { BrowserImportInfo, BrowserTabInfo, BrowserUiState } from '@shared/types'
|
||||
import {
|
||||
BrowserCdpTimeoutError,
|
||||
BROWSER_OBSERVE_TIMEOUT_MS,
|
||||
resolveBrowserObserveAxDepth,
|
||||
withBrowserCdpTimeout,
|
||||
type CdpResponse
|
||||
} from './browser-cdp'
|
||||
import {
|
||||
assertSafeBrowserUrl,
|
||||
browserObservationNameLimit,
|
||||
NAMELESS_ALLOWED_ROLES,
|
||||
prioritizeBrowserObservationCandidates,
|
||||
resolveBrowserDestination,
|
||||
resolveBrowserObserveMaxElements
|
||||
} from './browser-policy'
|
||||
|
||||
/**
|
||||
* 内置受管浏览器控制器。
|
||||
*
|
||||
* 架构(参考 Proma 的成熟实现,适配本应用单窗口/单聊天会话形态):
|
||||
* - 每个标签页 = 一个原生 WebContentsView,通过 contentView.addChildView 呈现;
|
||||
* 渲染层 BrowserSlot 量出占位矩形,经 IPC 同步边界(原生视图天然盖在 DOM 之上)。
|
||||
* - 页面观察/操作全部走 CDP(webContents.debugger):AX tree 快照 + ref 代际机制
|
||||
* + Input.dispatch* 真实输入事件;不在页面里执行模拟点击。
|
||||
* - 登录态隔离在 persist:aso-browser 分区;本地浏览器配置导入由 browser-profile.ts
|
||||
* 直接迁移分区目录数据(Cookies/Local Storage + 密钥移植)。
|
||||
*/
|
||||
|
||||
const PARTITION = 'persist:aso-browser'
|
||||
const NEW_TAB_URL = 'https://www.google.com/'
|
||||
const MAX_TABS = 12
|
||||
const MAX_FILL_CHARS = 10_000
|
||||
const MAX_SCRIPT_CHARS = 20_000
|
||||
const MAX_EXTRACT_CHARS = 50_000
|
||||
|
||||
interface RefEntry {
|
||||
backendNodeId: number
|
||||
generation: number
|
||||
label: string
|
||||
editable: boolean
|
||||
}
|
||||
|
||||
interface TabRecord {
|
||||
tabId: string
|
||||
view: WebContentsView
|
||||
createdAt: number
|
||||
url: string
|
||||
title: string
|
||||
loading: boolean
|
||||
/** ref 表与代际号:每次观察/导航后旧 ref 失效 */
|
||||
refs: Map<string, RefEntry>
|
||||
generation: number
|
||||
/** 同一 tab 的 Agent 操作串行化,避免 CDP 命令交错 */
|
||||
commandTail: Promise<unknown>
|
||||
lastLayoutRevision: number
|
||||
attached: boolean
|
||||
}
|
||||
|
||||
interface BrowserSettings {
|
||||
riskAcknowledged: boolean
|
||||
imported?: BrowserImportInfo | null
|
||||
}
|
||||
|
||||
export interface BrowserObservation {
|
||||
tabId: string
|
||||
url: string
|
||||
title: string
|
||||
generation: number
|
||||
elements: Array<{ ref: string; role: string; name: string; editable: boolean }>
|
||||
}
|
||||
|
||||
export interface BrowserWaitCondition {
|
||||
kind: 'url' | 'text' | 'selector'
|
||||
value: string
|
||||
}
|
||||
|
||||
function textValue(v: unknown): string {
|
||||
if (v && typeof v === 'object') {
|
||||
const value = (v as { value?: unknown }).value
|
||||
if (typeof value === 'string') return value
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function isEditableAxNode(node: Record<string, unknown>): boolean {
|
||||
const role = textValue(node.role).toLowerCase()
|
||||
if (role === 'textfield' || role === 'textbox' || role === 'searchbox') return true
|
||||
const props = Array.isArray(node.properties) ? (node.properties as Array<Record<string, unknown>>) : []
|
||||
for (const p of props) {
|
||||
if (String(p.name) === 'editable') {
|
||||
const inner = p.value as { value?: unknown } | undefined
|
||||
if (inner && typeof inner.value === 'boolean') return inner.value
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** 导航键 → CDP key 事件参数(rawKeyDown + VK code 才会触发默认行为:滚动/提交/焦点移动) */
|
||||
const KEY_EVENTS: Record<string, { key: string; code: string; vk: number }> = {
|
||||
enter: { key: 'Enter', code: 'Enter', vk: 13 },
|
||||
tab: { key: 'Tab', code: 'Tab', vk: 9 },
|
||||
escape: { key: 'Escape', code: 'Escape', vk: 27 },
|
||||
backspace: { key: 'Backspace', code: 'Backspace', vk: 8 },
|
||||
delete: { key: 'Delete', code: 'Delete', vk: 46 },
|
||||
pageup: { key: 'PageUp', code: 'PageUp', vk: 33 },
|
||||
pagedown: { key: 'PageDown', code: 'PageDown', vk: 34 },
|
||||
home: { key: 'Home', code: 'Home', vk: 36 },
|
||||
end: { key: 'End', code: 'End', vk: 35 },
|
||||
arrowup: { key: 'ArrowUp', code: 'ArrowUp', vk: 38 },
|
||||
arrowdown: { key: 'ArrowDown', code: 'ArrowDown', vk: 40 },
|
||||
arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft', vk: 37 },
|
||||
arrowright: { key: 'ArrowRight', code: 'ArrowRight', vk: 39 },
|
||||
space: { key: ' ', code: 'Space', vk: 32 }
|
||||
}
|
||||
|
||||
export class BrowserController {
|
||||
private owner: BrowserWindow | null = null
|
||||
private browserSession: Session
|
||||
private tabs = new Map<string, TabRecord>()
|
||||
private activeTabId: string | null = null
|
||||
private agentTabId: string | null = null
|
||||
private listeners = new Set<(state: BrowserUiState) => void>()
|
||||
private traceText: string | null = null
|
||||
private settings: BrowserSettings = { riskAcknowledged: false }
|
||||
|
||||
constructor() {
|
||||
this.browserSession = electronSession.fromPartition(PARTITION)
|
||||
// 诚实的 UA:保留 Chromium token 保证站点兼容,去掉 Electron 标识
|
||||
this.browserSession.setUserAgent(
|
||||
this.browserSession.getUserAgent().replace(/Electron\/[\d.]+/, `InkreachAgent/${app.getVersion()}`)
|
||||
)
|
||||
// 网页权限请求(摄像头/定位等)默认拒绝
|
||||
this.browserSession.setPermissionRequestHandler((_wc, _permission, callback) => callback(false))
|
||||
// 下载留在受管浏览器内,落到系统下载目录(重名自动加序号)
|
||||
this.browserSession.on('will-download', (_event, item) => {
|
||||
const dir = app.getPath('downloads')
|
||||
const base = item.getFilename() || 'download'
|
||||
const dot = base.lastIndexOf('.')
|
||||
const stem = dot > 0 ? base.slice(0, dot) : base
|
||||
const ext = dot > 0 ? base.slice(dot) : ''
|
||||
let target = join(dir, base)
|
||||
for (let i = 1; existsSync(target) && i < 100; i++) {
|
||||
target = join(dir, `${stem} (${i})${ext}`)
|
||||
}
|
||||
item.setSavePath(target)
|
||||
})
|
||||
this.settings = this.loadSettings()
|
||||
}
|
||||
|
||||
// ---- 设置持久化 ----
|
||||
|
||||
private get settingsFile(): string {
|
||||
return join(app.getPath('userData'), 'aso-browser.json')
|
||||
}
|
||||
|
||||
private loadSettings(): BrowserSettings {
|
||||
try {
|
||||
const raw = JSON.parse(readFileSync(this.settingsFile, 'utf8')) as Partial<BrowserSettings>
|
||||
return {
|
||||
riskAcknowledged: raw.riskAcknowledged === true,
|
||||
imported: raw.imported ?? null
|
||||
}
|
||||
} catch {
|
||||
return { riskAcknowledged: false }
|
||||
}
|
||||
}
|
||||
|
||||
private saveSettings(): void {
|
||||
try {
|
||||
writeFileSync(this.settingsFile, JSON.stringify(this.settings, null, 2), 'utf8')
|
||||
} catch (err) {
|
||||
console.error('[browser] save settings failed:', err)
|
||||
}
|
||||
}
|
||||
|
||||
/** 配置导入完成后由 browser-profile.ts 回写摘要 */
|
||||
setImportedInfo(info: BrowserImportInfo | null): void {
|
||||
this.settings.imported = info
|
||||
this.saveSettings()
|
||||
this.emitState()
|
||||
}
|
||||
|
||||
get importedInfo(): BrowserImportInfo | null {
|
||||
return this.settings.imported ?? null
|
||||
}
|
||||
|
||||
/** 当前是否有活跃标签页(导入配置前必须没有) */
|
||||
get hasTabs(): boolean {
|
||||
return this.tabs.size > 0
|
||||
}
|
||||
|
||||
// ---- 窗口与状态广播 ----
|
||||
|
||||
setOwnerWindow(win: BrowserWindow): void {
|
||||
this.owner = win
|
||||
}
|
||||
|
||||
handleWindowClosed(): void {
|
||||
for (const tab of this.tabs.values()) {
|
||||
try {
|
||||
if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach()
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
try {
|
||||
tab.view.setVisible(false)
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
}
|
||||
this.tabs.clear()
|
||||
this.activeTabId = null
|
||||
this.agentTabId = null
|
||||
this.owner = null
|
||||
}
|
||||
|
||||
onState(fn: (state: BrowserUiState) => void): () => void {
|
||||
this.listeners.add(fn)
|
||||
return () => this.listeners.delete(fn)
|
||||
}
|
||||
|
||||
private buildState(): BrowserUiState {
|
||||
const tabs: BrowserTabInfo[] = [...this.tabs.values()]
|
||||
.sort((a, b) => a.createdAt - b.createdAt)
|
||||
.map((t) => ({ tabId: t.tabId, url: t.url, title: t.title, loading: t.loading }))
|
||||
return {
|
||||
open: tabs.length > 0,
|
||||
riskAcknowledged: this.settings.riskAcknowledged,
|
||||
tabs,
|
||||
activeTabId: this.activeTabId,
|
||||
trace: this.traceText,
|
||||
imported: this.settings.imported ?? null
|
||||
}
|
||||
}
|
||||
|
||||
getState(): BrowserUiState {
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
private emitState(): void {
|
||||
const state = this.buildState()
|
||||
for (const fn of this.listeners) {
|
||||
try {
|
||||
fn(state)
|
||||
} catch (err) {
|
||||
console.error('[browser] state listener error:', err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private trace(tab: TabRecord, action: string, detail: string): void {
|
||||
this.traceText = `[${action}] ${detail}`
|
||||
void tab
|
||||
this.emitState()
|
||||
}
|
||||
|
||||
// ---- 风险告知门禁 ----
|
||||
|
||||
acknowledgeRisk(): void {
|
||||
this.settings.riskAcknowledged = true
|
||||
this.saveSettings()
|
||||
this.emitState()
|
||||
}
|
||||
|
||||
private assertRiskAcknowledged(): void {
|
||||
if (!this.settings.riskAcknowledged) {
|
||||
throw new Error('内置浏览器首次使用尚未确认风险告知:请在应用右侧浏览器面板中阅读并点击「我已了解,开始使用」后重试。')
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 会话与标签页 ----
|
||||
|
||||
private ensureBrowser(): TabRecord {
|
||||
if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪,无法使用内置浏览器。')
|
||||
if (this.tabs.size === 0) this.createTab(NEW_TAB_URL)
|
||||
const active = this.activeTabId ? this.tabs.get(this.activeTabId) : null
|
||||
const fallback = this.tabs.values().next().value
|
||||
const tab = active ?? fallback
|
||||
if (!tab) throw new Error('浏览器标签页不可用。')
|
||||
this.emitState()
|
||||
return tab
|
||||
}
|
||||
|
||||
private invalidateTabDocument(tab: TabRecord): void {
|
||||
tab.generation += 1
|
||||
tab.refs.clear()
|
||||
}
|
||||
|
||||
private createTab(initialUrl: string, viewOptions?: Electron.WebContentsViewConstructorOptions): TabRecord {
|
||||
if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪,无法创建浏览器标签。')
|
||||
if (this.tabs.size >= MAX_TABS) throw new Error(`浏览器标签页已达上限(${MAX_TABS}),请先关闭不需要的标签。`)
|
||||
const tabId = `tab-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
||||
const view = new WebContentsView(
|
||||
viewOptions ?? {
|
||||
webPreferences: {
|
||||
partition: PARTITION,
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
webSecurity: true
|
||||
}
|
||||
}
|
||||
)
|
||||
const tab: TabRecord = {
|
||||
tabId,
|
||||
view,
|
||||
createdAt: Date.now(),
|
||||
url: '',
|
||||
title: '新标签页',
|
||||
loading: false,
|
||||
refs: new Map(),
|
||||
generation: 0,
|
||||
commandTail: Promise.resolve(),
|
||||
lastLayoutRevision: 0,
|
||||
attached: false
|
||||
}
|
||||
view.setVisible(false)
|
||||
|
||||
// popup 收编为受管同级标签(必须同步返回 webContents,故提前建好 TabRecord)
|
||||
view.webContents.setWindowOpenHandler(({ url }) => {
|
||||
try {
|
||||
assertSafeBrowserUrl(url)
|
||||
} catch {
|
||||
this.trace(tab, 'popup', `已阻止不安全的新窗口: ${url.slice(0, 80)}`)
|
||||
return { action: 'deny' }
|
||||
}
|
||||
if (this.tabs.get(tabId) !== tab || this.tabs.size >= MAX_TABS) return { action: 'deny' }
|
||||
return {
|
||||
action: 'allow',
|
||||
outlivesOpener: true,
|
||||
createWindow: (options) => {
|
||||
const popup = this.createTab(url, options)
|
||||
this.selectTab(popup.tabId)
|
||||
this.trace(popup, 'popup', '网页打开了新标签页')
|
||||
return popup.view.webContents
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const updateNav = (): void => {
|
||||
const wc = view.webContents
|
||||
if (wc.isDestroyed()) return
|
||||
try {
|
||||
tab.url = wc.getURL()
|
||||
tab.title = wc.getTitle() || tab.url
|
||||
tab.loading = wc.isLoading()
|
||||
} catch {
|
||||
// 竞态时忽略
|
||||
}
|
||||
this.emitState()
|
||||
}
|
||||
view.webContents.on('did-start-loading', () => {
|
||||
tab.loading = true
|
||||
this.invalidateTabDocument(tab)
|
||||
this.emitState()
|
||||
})
|
||||
view.webContents.on('did-stop-loading', updateNav)
|
||||
view.webContents.on('page-title-updated', updateNav)
|
||||
view.webContents.on('did-navigate', () => {
|
||||
this.invalidateTabDocument(tab)
|
||||
updateNav()
|
||||
})
|
||||
view.webContents.on('did-navigate-in-page', () => {
|
||||
this.invalidateTabDocument(tab)
|
||||
updateNav()
|
||||
})
|
||||
view.webContents.on('destroyed', () => {
|
||||
this.tabs.delete(tabId)
|
||||
this.removeView(view)
|
||||
if (this.activeTabId === tabId) this.activeTabId = this.tabs.keys().next().value ?? null
|
||||
if (this.agentTabId === tabId) this.agentTabId = this.activeTabId
|
||||
this.emitState()
|
||||
})
|
||||
|
||||
try {
|
||||
view.webContents.debugger.attach('1.3')
|
||||
} catch (err) {
|
||||
console.warn('[browser] CDP attach 失败:', err)
|
||||
}
|
||||
this.tabs.set(tabId, tab)
|
||||
if (!this.activeTabId) this.activeTabId = tabId
|
||||
// popup 场景(带 viewOptions)由 Electron 的 createWindow 机制自行导航,避免双重加载
|
||||
if (initialUrl && !viewOptions) void view.webContents.loadURL(assertSafeBrowserUrl(initialUrl)).catch(() => undefined)
|
||||
return tab
|
||||
}
|
||||
|
||||
private removeView(view: WebContentsView): void {
|
||||
try {
|
||||
this.owner?.contentView.removeChildView(view)
|
||||
} catch {
|
||||
// 窗口已销毁
|
||||
}
|
||||
try {
|
||||
view.webContents.close()
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
}
|
||||
|
||||
listTabs(): BrowserUiState {
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
newTab(url?: string, byAgent = true): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.createTab(url ? resolveBrowserDestination(url) : NEW_TAB_URL)
|
||||
this.activeTabId = tab.tabId
|
||||
if (byAgent) this.agentTabId = tab.tabId
|
||||
this.emitState()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
selectTab(tabId: string): BrowserUiState {
|
||||
const tab = this.tabs.get(tabId)
|
||||
if (!tab) throw new Error(`标签页不存在: ${tabId}`)
|
||||
this.activeTabId = tabId
|
||||
try {
|
||||
tab.view.webContents.focus()
|
||||
} catch {
|
||||
// 页面可能正在加载
|
||||
}
|
||||
this.emitState()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
closeTab(tabId?: string): BrowserUiState {
|
||||
const id = tabId ?? this.activeTabId
|
||||
if (!id) return this.buildState()
|
||||
const tab = this.tabs.get(id)
|
||||
if (!tab) return this.buildState()
|
||||
try {
|
||||
if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach()
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
this.removeView(tab.view)
|
||||
this.tabs.delete(id)
|
||||
if (this.activeTabId === id) this.activeTabId = this.tabs.keys().next().value ?? null
|
||||
if (this.agentTabId === id) this.agentTabId = this.activeTabId
|
||||
this.emitState()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
/** 关闭整个受管浏览器(销毁全部标签;导入配置前也会走这里) */
|
||||
close(): BrowserUiState {
|
||||
for (const tabId of [...this.tabs.keys()]) this.closeTab(tabId)
|
||||
this.agentTabId = null
|
||||
this.traceText = null
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
// ---- 渲染层布局同步 ----
|
||||
|
||||
applyLayout(layout: { tabId: string; visible: boolean; bounds: Rectangle; revision: number }): void {
|
||||
const tab = this.tabs.get(layout.tabId)
|
||||
if (!tab || !this.owner || this.owner.isDestroyed()) return
|
||||
if (layout.revision <= tab.lastLayoutRevision) return
|
||||
tab.lastLayoutRevision = layout.revision
|
||||
try {
|
||||
if (layout.visible && layout.bounds.width > 4 && layout.bounds.height > 4) {
|
||||
if (!tab.attached) {
|
||||
this.owner.contentView.addChildView(tab.view)
|
||||
tab.attached = true
|
||||
}
|
||||
tab.view.setBounds(layout.bounds)
|
||||
tab.view.setVisible(true)
|
||||
} else {
|
||||
tab.view.setVisible(false)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[browser] apply layout failed:', err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 用户导航(地址栏/按钮) ----
|
||||
|
||||
navigateDisplay(input: string): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
void tab.view.webContents.loadURL(resolveBrowserDestination(input)).catch(() => undefined)
|
||||
this.trace(tab, 'navigate', `打开 ${input.slice(0, 80)}`)
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
goBack(): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
if (tab.view.webContents.navigationHistory.canGoBack()) tab.view.webContents.navigationHistory.goBack()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
goForward(): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
if (tab.view.webContents.navigationHistory.canGoForward()) tab.view.webContents.navigationHistory.goForward()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
reload(): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
tab.view.webContents.reload()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
// ---- CDP 基础 ----
|
||||
|
||||
private cdp(tab: TabRecord, method: string, params?: Record<string, unknown>, timeoutMs?: number): Promise<CdpResponse> {
|
||||
const debuggerClient = tab.view.webContents.debugger
|
||||
if (!debuggerClient.isAttached()) throw new Error('浏览器调试通道不可用,请重新打开该标签页。')
|
||||
return withBrowserCdpTimeout(() => debuggerClient.sendCommand(method, params) as Promise<CdpResponse>, method, timeoutMs).catch(
|
||||
(error: unknown) => {
|
||||
if (error instanceof BrowserCdpTimeoutError) this.recoverDebugger(tab, method)
|
||||
throw error
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/** 超时后重连 debugger,避免一个卡死页面让后续浏览器调用全部失败 */
|
||||
private recoverDebugger(tab: TabRecord, method: string): void {
|
||||
try {
|
||||
const debuggerClient = tab.view.webContents.debugger
|
||||
if (debuggerClient.isAttached()) debuggerClient.detach()
|
||||
debuggerClient.attach('1.3')
|
||||
console.warn(`[browser] ${method} 超时,已重连调试通道`)
|
||||
} catch {
|
||||
// 页面已关闭
|
||||
}
|
||||
}
|
||||
|
||||
/** Agent 的 tab 内操作串行化(用户操作不排队) */
|
||||
private runAgentOp<T>(tab: TabRecord, fn: () => Promise<T>): Promise<T> {
|
||||
const next = tab.commandTail.then(fn, fn)
|
||||
tab.commandTail = next.catch(() => undefined)
|
||||
return next
|
||||
}
|
||||
|
||||
private getAgentTab(tabId?: string): TabRecord {
|
||||
if (tabId) {
|
||||
const explicit = this.tabs.get(tabId)
|
||||
if (!explicit) throw new Error('目标标签页不存在,请先 browser_list_tabs 并用 browser_new_tab/browser_select_tab 选择。')
|
||||
return explicit
|
||||
}
|
||||
const id = this.agentTabId ?? this.activeTabId
|
||||
const tab = id ? this.tabs.get(id) : null
|
||||
if (tab) return tab
|
||||
// Agent 首次操作时还没有任何标签页:自动开一个工作页
|
||||
return this.ensureBrowser()
|
||||
}
|
||||
|
||||
private assertCurrentDocument(tab: TabRecord, generation: number): void {
|
||||
if (tab.generation !== generation) {
|
||||
throw new Error('页面已变化(导航或重渲染),旧元素引用失效:请重新 browser_observe 后再操作。')
|
||||
}
|
||||
}
|
||||
|
||||
private resolveRef(tab: TabRecord, ref: string): RefEntry {
|
||||
const entry = tab.refs.get(ref)
|
||||
if (!entry || entry.generation !== tab.generation) {
|
||||
throw new Error('元素引用已失效,请先重新调用 browser_observe。')
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
/** AX ref → 视口中心坐标(先滚动到可见再读 box;懒加载列表滚动后坐标会变,不能复用旧值) */
|
||||
private async centerForRef(tab: TabRecord, ref: string, generation: number): Promise<{ x: number; y: number }> {
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
const { backendNodeId } = this.resolveRef(tab, ref)
|
||||
await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
const box = await this.cdp(tab, 'DOM.getBoxModel', { backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
const model = box.model as Record<string, unknown> | undefined
|
||||
const quad = Array.isArray(model?.content) ? (model?.content as unknown[]) : []
|
||||
if (quad.length < 8 || !quad.every((v) => typeof v === 'number')) {
|
||||
throw new Error('目标元素当前不可点击,请重新观察页面。')
|
||||
}
|
||||
const n = quad as number[]
|
||||
return {
|
||||
x: (n[0] + n[2] + n[4] + n[6]) / 4,
|
||||
y: (n[1] + n[3] + n[5] + n[7]) / 4
|
||||
}
|
||||
}
|
||||
|
||||
/** 点击前在页面上短暂高亮目标,用户能看清 Agent 正在操作什么 */
|
||||
private async highlightAgentTarget(tab: TabRecord, backendNodeId: number): Promise<void> {
|
||||
try {
|
||||
const resolved = await this.cdp(tab, 'DOM.resolveNode', { backendNodeId })
|
||||
const objectId = (resolved.object as { objectId?: string } | undefined)?.objectId
|
||||
if (!objectId) return
|
||||
await this.cdp(tab, 'Runtime.callFunctionOn', {
|
||||
functionDeclaration:
|
||||
'function(){const r=this.getBoundingClientRect();const d=document.createElement("div");' +
|
||||
'd.style.cssText=`position:fixed;left:${r.left-2}px;top:${r.top-2}px;width:${r.width+4}px;height:${r.height+4}px;`+' +
|
||||
'"outline:2px solid #f59e0b;outline-offset:1px;pointer-events:none;z-index:2147483647;";' +
|
||||
'document.documentElement.appendChild(d);setTimeout(()=>d.remove(),700);return true}',
|
||||
objectId,
|
||||
returnByValue: true
|
||||
})
|
||||
} catch {
|
||||
// 高亮失败不影响操作
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Agent 工具实现 ----
|
||||
|
||||
async navigate(url: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
this.agentTabId = tab.tabId
|
||||
this.activeTabId = tab.tabId
|
||||
const destination = resolveBrowserDestination(url)
|
||||
await tab.view.webContents.loadURL(destination)
|
||||
this.trace(tab, 'navigate', `打开 ${destination.slice(0, 80)}`)
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
private parseAxNodes(response: CdpResponse): Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> {
|
||||
const nodes = Array.isArray(response.nodes) ? (response.nodes as Array<Record<string, unknown>>) : []
|
||||
const candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> = []
|
||||
for (const node of nodes) {
|
||||
if (!node) continue
|
||||
const backendNodeId = typeof node.backendDOMNodeId === 'number' ? node.backendDOMNodeId : 0
|
||||
const role = textValue(node.role).toLowerCase()
|
||||
const name = textValue(node.name)
|
||||
const editable = isEditableAxNode(node)
|
||||
if (!backendNodeId || !role) continue
|
||||
if (!name && !editable && !NAMELESS_ALLOWED_ROLES.has(role)) continue
|
||||
candidates.push({ backendNodeId, role, name: name.slice(0, browserObservationNameLimit(role)), editable })
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
private buildRefs(tab: TabRecord, candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }>): BrowserObservation['elements'] {
|
||||
tab.generation += 1
|
||||
tab.refs.clear()
|
||||
const elements: BrowserObservation['elements'] = []
|
||||
for (const candidate of candidates) {
|
||||
const ref = `r${tab.generation}-${elements.length + 1}`
|
||||
tab.refs.set(ref, {
|
||||
backendNodeId: candidate.backendNodeId,
|
||||
generation: tab.generation,
|
||||
label: candidate.name ? `${candidate.role}「${candidate.name.slice(0, 80)}」` : candidate.role,
|
||||
editable: candidate.editable
|
||||
})
|
||||
elements.push({ ref, role: candidate.role, name: candidate.name, editable: candidate.editable })
|
||||
}
|
||||
return elements
|
||||
}
|
||||
|
||||
async observe(tabId?: string, requestedMaxElements?: number): Promise<BrowserObservation> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
this.agentTabId = tab.tabId
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const maxElements = resolveBrowserObserveMaxElements(requestedMaxElements)
|
||||
const depth = resolveBrowserObserveAxDepth(maxElements)
|
||||
const response = await this.cdp(
|
||||
tab,
|
||||
'Accessibility.getFullAXTree',
|
||||
{ depth },
|
||||
BROWSER_OBSERVE_TIMEOUT_MS
|
||||
)
|
||||
const selected = prioritizeBrowserObservationCandidates(this.parseAxNodes(response), maxElements)
|
||||
const elements = this.buildRefs(tab, selected)
|
||||
this.trace(tab, 'observe', `读取到 ${elements.length}/${maxElements} 个元素(AX 深度 ${depth})`)
|
||||
return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements }
|
||||
})
|
||||
}
|
||||
|
||||
async find(
|
||||
query: { role?: string; name?: string; exact?: boolean; maxResults?: number },
|
||||
tabId?: string
|
||||
): Promise<BrowserObservation> {
|
||||
const role = query.role?.trim().toLowerCase()
|
||||
const name = query.name?.trim().toLowerCase()
|
||||
const maxResults = query.maxResults ?? 20
|
||||
if (!role && !name) throw new Error('语义定位至少需要 role 或 name。')
|
||||
if (!Number.isInteger(maxResults) || maxResults < 1 || maxResults > 50) throw new Error('maxResults 必须是 1 到 50 的整数。')
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
this.agentTabId = tab.tabId
|
||||
return this.runAgentOp(tab, async () => {
|
||||
// 过滤后再套观察配额:Find 能找到被紧凑 Observe 截掉的目标
|
||||
const response = await this.cdp(
|
||||
tab,
|
||||
'Accessibility.getFullAXTree',
|
||||
{ depth: resolveBrowserObserveAxDepth(MAX_BROWSER_OBSERVE_DEPTH_FALLBACK) },
|
||||
BROWSER_OBSERVE_TIMEOUT_MS
|
||||
)
|
||||
const candidates = this.parseAxNodes(response).filter((candidate) => {
|
||||
const roleMatches = !role || candidate.role === role
|
||||
const candidateName = candidate.name.toLowerCase()
|
||||
const nameMatches = !name || (query.exact ? candidateName === name : candidateName.includes(name))
|
||||
return roleMatches && nameMatches
|
||||
})
|
||||
const elements = this.buildRefs(tab, candidates.slice(0, maxResults))
|
||||
this.trace(tab, 'find', `语义定位到 ${elements.length} 个元素(${role ?? name})`)
|
||||
return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements }
|
||||
})
|
||||
}
|
||||
|
||||
private async clickRef(tab: TabRecord, ref: string, generation: number): Promise<RefEntry> {
|
||||
const target = this.resolveRef(tab, ref)
|
||||
const { x, y } = await this.centerForRef(tab, ref, generation)
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 })
|
||||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 })
|
||||
return target
|
||||
}
|
||||
|
||||
async click(ref: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = await this.clickRef(tab, ref, generation)
|
||||
this.trace(tab, 'click', `点击 ${target.label}`)
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
/** 点击 + 等待合并为一次串行操作(已知点击后预期状态时优先使用) */
|
||||
async act(ref: string, waitFor: BrowserWaitCondition | undefined, timeoutMs = 10_000, tabId?: string): Promise<{ state: BrowserUiState; matched: boolean }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = await this.clickRef(tab, ref, generation)
|
||||
this.trace(tab, 'click', `点击 ${target.label}`)
|
||||
if (!waitFor) return { state: this.buildState(), matched: true }
|
||||
const matched = await this.waitForInternal(tab, waitFor, timeoutMs)
|
||||
return { state: this.buildState(), matched }
|
||||
})
|
||||
}
|
||||
|
||||
async hover(ref: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = this.resolveRef(tab, ref)
|
||||
const { x, y } = await this.centerForRef(tab, ref, generation)
|
||||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y })
|
||||
this.trace(tab, 'hover', `悬停 ${target.label}`)
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
async fill(ref: string, text: string, tabId?: string): Promise<BrowserUiState> {
|
||||
if (text.length > MAX_FILL_CHARS) throw new Error(`单次输入不能超过 ${MAX_FILL_CHARS} 个字符。`)
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = this.resolveRef(tab, ref)
|
||||
if (!target.editable) throw new Error('目标元素不是可编辑字段,请重新观察后选择 input、textarea 或 contenteditable。')
|
||||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||||
await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.cdp(tab, 'DOM.focus', { backendNodeId: target.backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
// 全选(Ctrl/Cmd+A)后 insertText 整段替换,不逐字符模拟
|
||||
const selectAllModifier = process.platform === 'darwin' ? 4 : 2
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', {
|
||||
type: 'keyDown',
|
||||
key: 'a',
|
||||
code: 'KeyA',
|
||||
windowsVirtualKeyCode: 65,
|
||||
modifiers: selectAllModifier
|
||||
})
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', {
|
||||
type: 'keyUp',
|
||||
key: 'a',
|
||||
code: 'KeyA',
|
||||
windowsVirtualKeyCode: 65,
|
||||
modifiers: selectAllModifier
|
||||
})
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.cdp(tab, 'Input.insertText', { text })
|
||||
this.trace(tab, 'fill', `在 ${target.label} 输入 ${Array.from(text).length} 个字符`)
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
async press(key: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const normalized = key.trim().toLowerCase().replace(/\s+/g, '')
|
||||
const keyEvent = KEY_EVENTS[normalized]
|
||||
if (keyEvent) {
|
||||
const event = { key: keyEvent.key, code: keyEvent.code, windowsVirtualKeyCode: keyEvent.vk }
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'rawKeyDown', ...event })
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'keyUp', ...event })
|
||||
this.trace(tab, 'press', `按下 ${keyEvent.key}`)
|
||||
} else {
|
||||
await this.cdp(tab, 'Input.insertText', { text: key })
|
||||
this.trace(tab, 'press', `输入 ${Array.from(key).length} 个字符`)
|
||||
}
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
async scroll(
|
||||
input: { deltaY?: number; position?: 'top' | 'bottom'; selector?: string },
|
||||
tabId?: string
|
||||
): Promise<{ before: { top: number; height: number }; after: { top: number; height: number }; moved: boolean }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : ''
|
||||
const position = input.position === 'top' || input.position === 'bottom' ? input.position : undefined
|
||||
const deltaY = Math.max(-50_000, Math.min(50_000, Math.round(input.deltaY ?? 600)))
|
||||
// 固定表达式:数据由 JSON 注入,不拼接用户脚本
|
||||
const expression =
|
||||
'((input)=>{const el=input.selector?document.querySelector(input.selector):(document.scrollingElement||document.documentElement);' +
|
||||
'if(!el)return {ok:false};const read=()=>({top:Math.round(el.scrollTop),height:Math.round(el.scrollHeight)});' +
|
||||
'const before=read();const top=input.position==="top"?0:input.position==="bottom"?Math.max(0,el.scrollHeight-el.clientHeight):el.scrollTop+input.deltaY;' +
|
||||
'el.scrollTo({top});const after=read();return {ok:true,before,after}})(' +
|
||||
JSON.stringify({ selector, position, deltaY }) +
|
||||
')'
|
||||
const response = await this.evaluateOnPage(tab, expression)
|
||||
const result = (response ?? {}) as { ok?: boolean; before?: { top: number; height: number }; after?: { top: number; height: number } }
|
||||
if (!result.ok || !result.before || !result.after) throw new Error('滚动目标不存在或不支持滚动,请检查 selector。')
|
||||
this.trace(tab, 'scroll', `滚动页面${selector ? `(${selector.slice(0, 40)})` : ''}`)
|
||||
return { before: result.before, after: result.after, moved: result.after.top !== result.before.top }
|
||||
})
|
||||
}
|
||||
|
||||
async extract(
|
||||
input: { selector?: string; maxChars?: number },
|
||||
tabId?: string
|
||||
): Promise<{ url: string; title: string; text: string; truncated: boolean }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : ''
|
||||
const maxChars = Math.min(MAX_EXTRACT_CHARS, Math.max(500, Math.round(input.maxChars ?? MAX_EXTRACT_CHARS)))
|
||||
const expression =
|
||||
'((input)=>{const root=input.selector?document.querySelector(input.selector):document.body;' +
|
||||
'if(!root)return null;const text=(root.innerText||"").replace(/\\n{3,}/g,"\\n\\n");' +
|
||||
'return {text:text.slice(0,input.maxChars),truncated:text.length>input.maxChars}})(' +
|
||||
JSON.stringify({ selector, maxChars }) +
|
||||
')'
|
||||
const response = await this.evaluateOnPage(tab, expression)
|
||||
const result = (response ?? {}) as { text?: string; truncated?: boolean } | null
|
||||
if (!result || typeof result.text !== 'string') throw new Error('提取目标不存在,请检查 selector。')
|
||||
this.trace(tab, 'extract', `提取正文 ${result.text.length} 字符${selector ? `(${selector.slice(0, 40)})` : ''}`)
|
||||
return { url: tab.url, title: tab.title, text: result.text, truncated: result.truncated === true }
|
||||
})
|
||||
}
|
||||
|
||||
async waitFor(condition: BrowserWaitCondition, timeoutMs = 10_000, tabId?: string): Promise<{ matched: boolean; url: string; title: string }> {
|
||||
if (!condition.value.trim()) throw new Error('等待条件不能为空。')
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs < 250 || timeoutMs > 30_000) throw new Error('等待超时必须在 250 到 30000 毫秒之间。')
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const matched = await this.waitForInternal(tab, condition, timeoutMs)
|
||||
this.trace(tab, 'wait', `${matched ? '已满足' : '超时未满足'}${condition.kind} 条件(${condition.value.slice(0, 40)})`)
|
||||
return { matched, url: tab.url, title: tab.title }
|
||||
})
|
||||
}
|
||||
|
||||
/** 轮询固定表达式(只读检查,不执行任意 JS) */
|
||||
private async waitForInternal(tab: TabRecord, condition: BrowserWaitCondition, timeoutMs: number): Promise<boolean> {
|
||||
const startedAt = Date.now()
|
||||
const payload = JSON.stringify(condition).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029')
|
||||
const expression =
|
||||
`(() => { const condition = ${payload}; try { ` +
|
||||
"if (condition.kind === 'url') return location.href.includes(condition.value); " +
|
||||
"if (condition.kind === 'text') return (document.body?.innerText || '').includes(condition.value); " +
|
||||
'return !!document.querySelector(condition.value); } catch { return false; } })()'
|
||||
while (Date.now() - startedAt <= timeoutMs) {
|
||||
const result = await this.evaluateOnPage(tab, expression)
|
||||
if (result === true) return true
|
||||
await new Promise((resolve) => setTimeout(resolve, 250))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** Agent 逃生舱:在页面上下文执行自写 JS(有长度上限,结果 JSON 化) */
|
||||
async evaluate(script: string, tabId?: string): Promise<unknown> {
|
||||
if (!script.trim()) throw new Error('脚本不能为空。')
|
||||
if (script.length > MAX_SCRIPT_CHARS) throw new Error(`脚本不能超过 ${MAX_SCRIPT_CHARS} 字符。`)
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const result = await this.evaluateOnPage(tab, script, true)
|
||||
this.trace(tab, 'js', `执行页面脚本 ${script.length} 字符`)
|
||||
return result
|
||||
})
|
||||
}
|
||||
|
||||
private async evaluateOnPage(tab: TabRecord, expression: string, userGesture = false): Promise<unknown> {
|
||||
const response = await this.cdp(tab, 'Runtime.evaluate', {
|
||||
expression,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
userGesture
|
||||
})
|
||||
const exception = response.exceptionDetails as { text?: string; exception?: { description?: string } } | undefined
|
||||
if (exception) {
|
||||
throw new Error(`页面脚本执行出错: ${exception.exception?.description || exception.text || '未知错误'}`)
|
||||
}
|
||||
const result = response.result as { type?: string; value?: unknown; description?: string } | undefined
|
||||
if (!result) return null
|
||||
if ('value' in result) return result.value ?? null
|
||||
return { type: result.type ?? 'unknown', description: result.description ?? null }
|
||||
}
|
||||
|
||||
async screenshot(tabId?: string): Promise<{ tabId: string; url: string; title: string; dataUrl: string }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const image = await tab.view.webContents.capturePage()
|
||||
this.trace(tab, 'screenshot', '截取当前页面')
|
||||
return { tabId: tab.tabId, url: tab.url, title: tab.title, dataUrl: image.toDataURL() }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_BROWSER_OBSERVE_DEPTH_FALLBACK = 400
|
||||
|
||||
let controllerInstance: BrowserController | null = null
|
||||
|
||||
/** 惰性单例:首次取用时才创建(session 相关 API 需在 app ready 后调用) */
|
||||
export function getBrowserController(): BrowserController {
|
||||
if (!controllerInstance) controllerInstance = new BrowserController()
|
||||
return controllerInstance
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* 受管浏览器的 URL 与观察策略。
|
||||
* 独立成模块便于单测与复用(渲染层不需要,主进程专用)。
|
||||
*/
|
||||
|
||||
const SEARCH_URL = 'https://www.google.com/search?q='
|
||||
|
||||
/** 导航白名单:普通 web、本地开发服务与 about:blank;file:/javascript:/data: 一律拒绝 */
|
||||
export function assertSafeBrowserUrl(input: string): string {
|
||||
const value = input.trim()
|
||||
if (!value) throw new Error('URL 不能为空。')
|
||||
if (value === 'about:blank') return value
|
||||
let parsed: URL
|
||||
try {
|
||||
parsed = new URL(value)
|
||||
} catch {
|
||||
throw new Error(`无法解析的 URL: ${value}`)
|
||||
}
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
||||
throw new Error(`不支持的地址协议(仅允许 http/https 与 about:blank): ${parsed.protocol}`)
|
||||
}
|
||||
return parsed.toString()
|
||||
}
|
||||
|
||||
function isLikelyUrl(value: string): boolean {
|
||||
const candidate = `http://${value}`
|
||||
try {
|
||||
const url = new URL(candidate)
|
||||
const host = url.hostname.toLowerCase()
|
||||
// 域名 / IP(v4/v6)/ localhost 与 *.localhost 视为直达地址
|
||||
const isHost =
|
||||
host === 'localhost' ||
|
||||
host.endsWith('.localhost') ||
|
||||
host.includes('.') ||
|
||||
host.includes(':')
|
||||
// 无空格且形如域名/IP/localhost
|
||||
return isHost && !/\s/.test(value)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** 输入归一化:URL/域名/IP/localhost 直达,普通文本走搜索 */
|
||||
export function resolveBrowserDestination(input: string): string {
|
||||
const value = input.trim()
|
||||
if (!value) throw new Error('地址不能为空。')
|
||||
if (value === 'about:blank') return value
|
||||
if (/^https?:\/\//i.test(value)) return assertSafeBrowserUrl(value)
|
||||
if (isLikelyUrl(value)) return assertSafeBrowserUrl(`https://${value}`)
|
||||
return SEARCH_URL + encodeURIComponent(value)
|
||||
}
|
||||
|
||||
// ---- Observe 观察策略 ----
|
||||
|
||||
export const DEFAULT_BROWSER_OBSERVE_MAX_ELEMENTS = 240
|
||||
export const MAX_BROWSER_OBSERVE_MAX_ELEMENTS = 400
|
||||
export const MIN_BROWSER_OBSERVE_MAX_ELEMENTS = 20
|
||||
|
||||
/** 可交互 AX role 白名单(占观察结果前 2/3 配额) */
|
||||
export const INTERACTIVE_AX_ROLES = new Set([
|
||||
'button',
|
||||
'link',
|
||||
'textfield',
|
||||
'textbox',
|
||||
'searchbox',
|
||||
'checkbox',
|
||||
'radiobutton',
|
||||
'combobox',
|
||||
'menuitem',
|
||||
'menuitemcheckbox',
|
||||
'menuitemradio',
|
||||
'tab',
|
||||
'option',
|
||||
'switch',
|
||||
'slider'
|
||||
])
|
||||
|
||||
/** 无名称也值得返回的 role(依赖 role 本身可定位) */
|
||||
export const NAMELESS_ALLOWED_ROLES = new Set(['button', 'textfield', 'textbox', 'link', 'checkbox', 'combobox'])
|
||||
|
||||
export function resolveBrowserObserveMaxElements(requested?: number): number {
|
||||
if (typeof requested !== 'number' || !Number.isFinite(requested)) return DEFAULT_BROWSER_OBSERVE_MAX_ELEMENTS
|
||||
return Math.min(MAX_BROWSER_OBSERVE_MAX_ELEMENTS, Math.max(MIN_BROWSER_OBSERVE_MAX_ELEMENTS, Math.round(requested)))
|
||||
}
|
||||
|
||||
export function isInteractiveAxRole(role: string): boolean {
|
||||
return INTERACTIVE_AX_ROLES.has(role.toLowerCase())
|
||||
}
|
||||
|
||||
export interface BrowserObservationCandidate {
|
||||
role: string
|
||||
name: string
|
||||
editable?: boolean
|
||||
}
|
||||
|
||||
/** 可交互元素优先(约 2/3 配额),其余语义上下文补足 */
|
||||
export function prioritizeBrowserObservationCandidates<T extends BrowserObservationCandidate>(
|
||||
candidates: readonly T[],
|
||||
maxElements: number
|
||||
): T[] {
|
||||
const interactiveLimit = Math.ceil(maxElements * (2 / 3))
|
||||
const interactive: T[] = []
|
||||
const context: T[] = []
|
||||
for (const candidate of candidates) {
|
||||
if (interactive.length < interactiveLimit && (isInteractiveAxRole(candidate.role) || candidate.editable)) {
|
||||
interactive.push(candidate)
|
||||
} else {
|
||||
context.push(candidate)
|
||||
}
|
||||
}
|
||||
const contextLimit = maxElements - interactive.length
|
||||
return [...interactive, ...context.slice(0, Math.max(0, contextLimit))]
|
||||
}
|
||||
|
||||
/** 不同 role 的可访问名称截断上限,避免长文本撑爆快照 */
|
||||
export function browserObservationNameLimit(role: string): number {
|
||||
if (role === 'link' || role === 'button' || role === 'menuitem') return 80
|
||||
if (role === 'heading') return 120
|
||||
return 160
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
import { app } from 'electron'
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
||||
import { homedir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import type { BrowserImportResult, BrowserProfileMeta, BrowserSource } from '@shared/types'
|
||||
import { getBrowserController } from './browser-controller'
|
||||
|
||||
/**
|
||||
* 本地浏览器配置导入(复用各网站登录态)。
|
||||
*
|
||||
* 实现思路(零解密依赖,纯文件迁移):
|
||||
* - Local Storage / IndexedDB 在 Chromium 里是明文 leveldb,直接拷入受管分区即可生效。
|
||||
* - Cookies 用 AES-GCM 加密,密钥经 OS 加密后存于浏览器根目录 Local State
|
||||
* (os_crypt.encrypted_key)。Windows 上该密钥由 DPAPI(用户级)包裹——同一
|
||||
* Windows 用户的任意进程都能解开,因此把这份密钥「移植」进本应用的 Local State,
|
||||
* 本应用内的 Chromium 即可解密拷贝过来的 Cookies(Chrome 127+ 的 app-bound
|
||||
* 加密 V20 除外,会明确提示不支持)。macOS/Linux 的密钥绑定钥匙串/KWallet,
|
||||
* Cookies 无法直接迁移,仅迁移 Local Storage。
|
||||
*/
|
||||
|
||||
/** Chromium 系浏览器的用户数据根目录(按平台) */
|
||||
const SOURCE_ROOTS: Array<{ id: string; name: string; win: string; mac: string; linux: string }> = [
|
||||
{
|
||||
id: 'chrome',
|
||||
name: 'Google Chrome',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'Google', 'Chrome', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'Google', 'Chrome'),
|
||||
linux: join(homedir(), '.config', 'google-chrome')
|
||||
},
|
||||
{
|
||||
id: 'edge',
|
||||
name: 'Microsoft Edge',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'Microsoft', 'Edge', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'Microsoft Edge'),
|
||||
linux: join(homedir(), '.config', 'microsoft-edge')
|
||||
},
|
||||
{
|
||||
id: 'brave',
|
||||
name: 'Brave',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'BraveSoftware', 'Brave-Browser', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'BraveSoftware', 'Brave-Browser'),
|
||||
linux: join(homedir(), '.config', 'BraveSoftware', 'Brave-Browser')
|
||||
},
|
||||
{
|
||||
id: 'chromium',
|
||||
name: 'Chromium',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'Chromium', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'Chromium'),
|
||||
linux: join(homedir(), '.config', 'chromium')
|
||||
}
|
||||
]
|
||||
|
||||
/** 受管浏览器分区目录(persist:aso-browser 的落盘位置) */
|
||||
function partitionDir(): string {
|
||||
return join(app.getPath('userData'), 'Partitions', 'aso-browser')
|
||||
}
|
||||
|
||||
function sourceRoot(id: string): string {
|
||||
const source = SOURCE_ROOTS.find((s) => s.id === id)
|
||||
if (!source) return ''
|
||||
return process.platform === 'win32' ? source.win : process.platform === 'darwin' ? source.mac : source.linux
|
||||
}
|
||||
|
||||
/** 读取 profile 的展示名(Preferences 的 profile.name,失败时用目录名) */
|
||||
function readProfileName(profileDir: string): string {
|
||||
try {
|
||||
const file = join(profileDir, 'Preferences')
|
||||
if (statSync(file).size > 8 * 1024 * 1024) return profileDir.split(/[\\/]/).pop() || profileDir
|
||||
const raw = JSON.parse(readFileSync(file, 'utf8')) as { profile?: { name?: string } }
|
||||
return raw.profile?.name?.trim() || profileDir.split(/[\\/]/).pop() || profileDir
|
||||
} catch {
|
||||
return profileDir.split(/[\\/]/).pop() || profileDir
|
||||
}
|
||||
}
|
||||
|
||||
/** 检测本机可导入的浏览器与 profile 列表 */
|
||||
export function detectBrowserSources(): BrowserSource[] {
|
||||
const sources: BrowserSource[] = []
|
||||
for (const source of SOURCE_ROOTS) {
|
||||
const root = sourceRoot(source.id)
|
||||
if (!root || !existsSync(root)) continue
|
||||
const profiles: BrowserProfileMeta[] = []
|
||||
try {
|
||||
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue
|
||||
if (entry.name.startsWith('.') || entry.name === 'System Profile' || entry.name === 'Guest Profile') continue
|
||||
const profileDir = join(root, entry.name)
|
||||
if (!existsSync(join(profileDir, 'Preferences'))) continue
|
||||
profiles.push({ dir: entry.name, name: readProfileName(profileDir) })
|
||||
}
|
||||
} catch {
|
||||
// 目录不可读则跳过该浏览器
|
||||
continue
|
||||
}
|
||||
if (profiles.length > 0) {
|
||||
sources.push({ id: source.id, name: source.name, userDataDir: root, profiles })
|
||||
}
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
function copyFileSafe(from: string, to: string): void {
|
||||
try {
|
||||
copyFileSync(from, to)
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code
|
||||
if (code === 'EBUSY' || code === 'EPERM' || code === 'EACCES') {
|
||||
throw new Error('源浏览器文件被占用:请完全退出对应的浏览器(含后台进程)后重试。')
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
/** 拷贝 leveldb 目录(跳过 LOCK/LOG 运行时文件),目标已存在时先清空 */
|
||||
function copyLevelDbDir(from: string, to: string): boolean {
|
||||
if (!existsSync(from)) return false
|
||||
const entries = readdirSync(from, { withFileTypes: true })
|
||||
const files = entries.filter((e) => e.isFile() && e.name !== 'LOCK' && !e.name.startsWith('LOG'))
|
||||
if (files.length === 0) return false
|
||||
rmSync(to, { recursive: true, force: true })
|
||||
mkdirSync(to, { recursive: true })
|
||||
for (const file of files) {
|
||||
copyFileSafe(join(from, file.name), join(to, file.name))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
interface SourceKeyInfo {
|
||||
key: string | null
|
||||
/** app-bound(V20)等不可移植场景的说明 */
|
||||
note?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取源浏览器 Local State 里的 os_crypt.encrypted_key 并判断可移植性。
|
||||
* Windows:前缀 DPAPI 可移植;V20 为 app-bound 加密(Chrome/Edge 127+),
|
||||
* 其他进程无法解开,明确降级。macOS/Linux:密钥在系统钥匙串/KWallet,跳过。
|
||||
*/
|
||||
function readSourceKey(userDataDir: string): SourceKeyInfo {
|
||||
if (process.platform !== 'win32') {
|
||||
return { key: null, note: '当前平台的浏览器 Cookie 密钥绑定系统钥匙串,暂不支持迁移 Cookie(LocalStorage 登录态仍会导入)。' }
|
||||
}
|
||||
try {
|
||||
const raw = JSON.parse(readFileSync(join(userDataDir, 'Local State'), 'utf8')) as {
|
||||
os_crypt?: { encrypted_key?: string }
|
||||
}
|
||||
const encoded = raw.os_crypt?.encrypted_key
|
||||
if (!encoded) return { key: null, note: '未找到源浏览器 Cookie 密钥,跳过 Cookie 迁移。' }
|
||||
const bytes = Buffer.from(encoded, 'base64')
|
||||
if (bytes.subarray(0, 3).toString('utf8') === 'V20') {
|
||||
return { key: null, note: '该浏览器使用了应用绑定加密(app-bound encryption),无法第三方迁移 Cookie;已导入 Local Storage,首次访问各站点仍需登录一次。' }
|
||||
}
|
||||
if (bytes.subarray(0, 5).toString('utf8') !== 'DPAPI') {
|
||||
return { key: null, note: '源浏览器 Cookie 密钥格式不受支持,跳过 Cookie 迁移。' }
|
||||
}
|
||||
return { key: encoded }
|
||||
} catch {
|
||||
return { key: null, note: '读取源浏览器 Local State 失败,跳过 Cookie 迁移。' }
|
||||
}
|
||||
}
|
||||
|
||||
/** 把源密钥移植进本应用根 Local State(首次会备份原文件) */
|
||||
function transplantKey(sourceKey: string): void {
|
||||
const appLocalState = join(app.getPath('userData'), 'Local State')
|
||||
const backup = `${appLocalState}.aso-backup`
|
||||
let doc: Record<string, unknown> = {}
|
||||
if (existsSync(appLocalState)) {
|
||||
try {
|
||||
doc = JSON.parse(readFileSync(appLocalState, 'utf8')) as Record<string, unknown>
|
||||
if (!existsSync(backup)) copyFileSync(appLocalState, backup)
|
||||
} catch {
|
||||
doc = {}
|
||||
}
|
||||
}
|
||||
const osCrypt = (doc.os_crypt ?? {}) as Record<string, unknown>
|
||||
osCrypt.encrypted_key = sourceKey
|
||||
doc.os_crypt = osCrypt
|
||||
writeFileSync(appLocalState, JSON.stringify(doc, null, 2), 'utf8')
|
||||
}
|
||||
|
||||
/** 导入指定浏览器的 profile 配置到受管浏览器分区 */
|
||||
export function importBrowserProfile(sourceId: string, profileDir: string): BrowserImportResult {
|
||||
const browser = getBrowserController()
|
||||
if (browser.hasTabs) {
|
||||
throw new Error('请先关闭内置浏览器的所有标签页,再导入浏览器配置。')
|
||||
}
|
||||
const root = sourceRoot(sourceId)
|
||||
if (!root || !existsSync(root)) throw new Error('未找到该浏览器的用户数据目录。')
|
||||
// 只接受该根目录下的直接子目录名,拒绝任意路径注入
|
||||
const dirName = profileDir.split(/[\\/]/).pop() ?? ''
|
||||
const sourceProfile = join(root, dirName)
|
||||
if (dirName !== profileDir || !existsSync(join(sourceProfile, 'Preferences'))) {
|
||||
throw new Error('profile 目录不存在。')
|
||||
}
|
||||
const sourceName = SOURCE_ROOTS.find((s) => s.id === sourceId)?.name ?? sourceId
|
||||
const target = partitionDir()
|
||||
|
||||
// 1) Local Storage(明文 leveldb,直接迁移)
|
||||
let localStorage = false
|
||||
try {
|
||||
localStorage = copyLevelDbDir(join(sourceProfile, 'Local Storage', 'leveldb'), join(target, 'Local Storage', 'leveldb'))
|
||||
} catch (err) {
|
||||
throw new Error(err instanceof Error ? err.message : `Local Storage 迁移失败: ${String(err)}`)
|
||||
}
|
||||
|
||||
// 2) Cookies + 密钥移植
|
||||
let cookies = false
|
||||
let cookieNote: string | undefined
|
||||
const keyInfo = readSourceKey(root)
|
||||
const cookieDb = existsSync(join(sourceProfile, 'Network', 'Cookies'))
|
||||
? join(sourceProfile, 'Network', 'Cookies')
|
||||
: existsSync(join(sourceProfile, 'Cookies'))
|
||||
? join(sourceProfile, 'Cookies')
|
||||
: null
|
||||
if (keyInfo.key && cookieDb) {
|
||||
try {
|
||||
transplantKey(keyInfo.key)
|
||||
mkdirSync(join(target, 'Network'), { recursive: true })
|
||||
copyFileSafe(cookieDb, join(target, 'Network', 'Cookies'))
|
||||
for (const suffix of ['-wal', '-shm']) {
|
||||
if (existsSync(`${cookieDb}${suffix}`)) copyFileSafe(`${cookieDb}${suffix}`, join(target, 'Network', `Cookies${suffix}`))
|
||||
}
|
||||
cookies = true
|
||||
} catch (err) {
|
||||
cookieNote = err instanceof Error ? err.message : 'Cookie 迁移失败。'
|
||||
}
|
||||
} else {
|
||||
cookieNote = keyInfo.note ?? '源浏览器没有可迁移的 Cookie 数据库。'
|
||||
}
|
||||
|
||||
const parts: string[] = []
|
||||
if (cookies) parts.push('Cookie(站点登录态)')
|
||||
if (localStorage) parts.push('LocalStorage')
|
||||
if (!cookies && !localStorage) {
|
||||
return { ok: false, cookies, localStorage, cookieNote, message: `未在 ${sourceName}(${dirName})中找到可迁移的登录数据。` }
|
||||
}
|
||||
|
||||
browser.setImportedInfo({
|
||||
browser: sourceName,
|
||||
profile: readProfileName(sourceProfile),
|
||||
at: new Date().toISOString(),
|
||||
cookies,
|
||||
localStorage,
|
||||
note: cookieNote
|
||||
})
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
cookies,
|
||||
localStorage,
|
||||
cookieNote,
|
||||
message: `已从 ${sourceName} 导入 ${parts.join(' + ')}。${cookieNote ?? ''}${
|
||||
cookies ? '若登录态未生效,重启应用后重试。' : ''
|
||||
}`
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { app, BrowserWindow, ipcMain, Menu, shell } from 'electron'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { registerIpc } from './ipc'
|
||||
import { getBrowserController } from './browser/browser-controller'
|
||||
|
||||
process.env.APP_ROOT = join(__dirname, '..')
|
||||
|
||||
@@ -71,9 +72,13 @@ async function createWindow(): Promise<void> {
|
||||
})
|
||||
})
|
||||
|
||||
// 窗口关闭时未决审批一律拒绝,避免 agent 永久挂起
|
||||
// 受管浏览器的原生 WebContentsView 挂在主窗口 contentView 上
|
||||
getBrowserController().setOwnerWindow(win)
|
||||
|
||||
// 窗口关闭时未决审批一律拒绝,避免 agent 永久挂起;受管浏览器同步清理
|
||||
win.on('closed', () => {
|
||||
agentService.approvals.settleAll(false)
|
||||
getBrowserController().handleWindowClosed()
|
||||
})
|
||||
|
||||
// 外部链接交给系统默认浏览器,不在应用内开新窗口
|
||||
|
||||
@@ -2,6 +2,8 @@ import { BrowserWindow, dialog, ipcMain, shell } from 'electron'
|
||||
import { mkdirSync } from 'node:fs'
|
||||
import { userInfo } from 'node:os'
|
||||
import { AgentService } from './pi/agent-service'
|
||||
import { getBrowserController } from './browser/browser-controller'
|
||||
import { detectBrowserSources, importBrowserProfile } from './browser/browser-profile'
|
||||
|
||||
/**
|
||||
* 注册 IPC 通道并把 agent 事件转发到所有渲染窗口。
|
||||
@@ -257,5 +259,70 @@ export function registerIpc(): AgentService {
|
||||
return service.pickAttachments(win ?? undefined)
|
||||
})
|
||||
|
||||
// ---- 内置受管浏览器 ----
|
||||
const browser = getBrowserController()
|
||||
// 面板状态广播(与 agent:event 同样转发到所有窗口)
|
||||
browser.onState((state) => {
|
||||
for (const win of BrowserWindow.getAllWindows()) {
|
||||
if (!win.isDestroyed()) win.webContents.send('browser:event', { type: 'state', state })
|
||||
}
|
||||
})
|
||||
|
||||
const browserPass = <T,>(fn: () => T): T => {
|
||||
try {
|
||||
return fn()
|
||||
} catch (err) {
|
||||
throw new Error(err instanceof Error ? err.message : String(err))
|
||||
}
|
||||
}
|
||||
|
||||
ipcMain.handle('browser:getState', () => browser.getState())
|
||||
ipcMain.handle('browser:navigate', (_e, input: unknown) =>
|
||||
browserPass(() => browser.navigateDisplay(typeof input === 'string' ? input : ''))
|
||||
)
|
||||
ipcMain.handle('browser:goBack', () => browserPass(() => browser.goBack()))
|
||||
ipcMain.handle('browser:goForward', () => browserPass(() => browser.goForward()))
|
||||
ipcMain.handle('browser:reload', () => browserPass(() => browser.reload()))
|
||||
ipcMain.handle('browser:newTab', (_e, url: unknown) =>
|
||||
browserPass(() => browser.newTab(typeof url === 'string' && url ? url : undefined, false))
|
||||
)
|
||||
ipcMain.handle('browser:selectTab', (_e, tabId: unknown) =>
|
||||
browserPass(() => browser.selectTab(typeof tabId === 'string' ? tabId : ''))
|
||||
)
|
||||
ipcMain.handle('browser:closeTab', (_e, tabId: unknown) =>
|
||||
browserPass(() => browser.closeTab(typeof tabId === 'string' ? tabId : undefined))
|
||||
)
|
||||
ipcMain.handle('browser:close', () => browserPass(() => browser.close()))
|
||||
ipcMain.handle('browser:acknowledgeRisk', () => browserPass(() => browser.acknowledgeRisk()))
|
||||
// 渲染层 BrowserSlot 量出的占位矩形(原生 WebContentsView 对齐到 DOM 布局)
|
||||
ipcMain.handle('browser:layout', (_e, layout: unknown) => {
|
||||
const l = layout as { tabId?: unknown; visible?: unknown; revision?: unknown; bounds?: unknown }
|
||||
const { tabId, visible, revision } = l
|
||||
const bounds = l.bounds as { x?: unknown; y?: unknown; width?: unknown; height?: unknown } | undefined
|
||||
const x = bounds?.x
|
||||
const y = bounds?.y
|
||||
const width = bounds?.width
|
||||
const height = bounds?.height
|
||||
if (
|
||||
typeof tabId !== 'string' ||
|
||||
typeof visible !== 'boolean' ||
|
||||
typeof revision !== 'number' ||
|
||||
typeof x !== 'number' ||
|
||||
typeof y !== 'number' ||
|
||||
typeof width !== 'number' ||
|
||||
typeof height !== 'number'
|
||||
) {
|
||||
throw new Error('invalid browser layout')
|
||||
}
|
||||
return browserPass(() => browser.applyLayout({ tabId, visible, revision, bounds: { x, y, width, height } }))
|
||||
})
|
||||
|
||||
// ---- 本地浏览器配置导入 ----
|
||||
ipcMain.handle('browser:detectSources', () => browserPass(() => detectBrowserSources()))
|
||||
ipcMain.handle('browser:importProfile', (_e, sourceId: unknown, profileDir: unknown) => {
|
||||
if (typeof sourceId !== 'string' || typeof profileDir !== 'string') throw new Error('invalid import source')
|
||||
return browserPass(() => importBrowserProfile(sourceId, profileDir))
|
||||
})
|
||||
|
||||
return service
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
} from '@earendil-works/pi-coding-agent'
|
||||
import { ApprovalService, createApprovalExtension } from './approval'
|
||||
import { demoTools, TOOLS_REQUIRING_APPROVAL } from '../tools/demo-tools'
|
||||
import { browserTools } from '../tools/browser-tools'
|
||||
import type {
|
||||
AgentUiEvent,
|
||||
AppSettings,
|
||||
@@ -269,7 +270,7 @@ export class AgentService {
|
||||
services,
|
||||
sessionManager,
|
||||
sessionStartEvent,
|
||||
customTools: demoTools
|
||||
customTools: [...demoTools, ...browserTools]
|
||||
})
|
||||
return { ...result, services, diagnostics: services.diagnostics }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
import { defineTool } from '@earendil-works/pi-coding-agent'
|
||||
import { Type } from '@earendil-works/pi-ai'
|
||||
import { getBrowserController } from '../browser/browser-controller'
|
||||
|
||||
/**
|
||||
* 内置受管浏览器工具(进程内一等工具,非 MCP)。
|
||||
*
|
||||
* 使用范式(与 Playwright MCP / Proma 一致的 AX 快照 + ref 体系):
|
||||
* 1. browser_observe / browser_find 拿到带 ref 的可交互元素快照;
|
||||
* 2. 用 ref 调 click / fill / act 等操作;页面变化后旧 ref 失效,必须重新观察;
|
||||
* 3. 语义结构足够时优先 observe,需要视觉验证时才 browser_screenshot。
|
||||
*/
|
||||
|
||||
const TAB_ID_PARAM = Type.Optional(
|
||||
Type.String({ description: '可选,目标标签页 ID。默认为 Agent 工作标签页(与用户可见面板同步)。' })
|
||||
)
|
||||
|
||||
export const browserTools = [
|
||||
defineTool({
|
||||
name: 'browser_navigate',
|
||||
label: '打开网页',
|
||||
description:
|
||||
'在内置受管浏览器中打开 URL 或搜索词。完整 URL、裸域名、localhost 与 IP 直达;普通文本用 Google 搜索。支持 about:blank 空白页。下载与弹窗会留在受管浏览器内。',
|
||||
promptSnippet: 'browser_navigate: 打开网页/搜索(URL 或关键词)',
|
||||
parameters: Type.Object({
|
||||
url: Type.String({ description: 'URL、域名或搜索关键词' }),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().navigate(params.url, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify(state.tabs.find((t) => t.tabId === state.activeTabId) ?? state) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_observe',
|
||||
label: '观察网页',
|
||||
description:
|
||||
'读取当前页面的 URL、标题与紧凑可访问性快照(元素带 ref,标注 editable)。默认返回 240 个元素(约 2/3 可交互优先)。每次 observe/find 或页面导航后旧 ref 全部失效,操作必须使用最新一次观察结果中的 ref。',
|
||||
promptSnippet: 'browser_observe: 读页面结构拿 ref(先观察再操作)',
|
||||
parameters: Type.Object({
|
||||
maxElements: Type.Optional(
|
||||
Type.Number({ minimum: 20, maximum: 400, description: '返回元素上限,默认 240;仅当目标不在快照中时才提高到 400' })
|
||||
),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const observation = await getBrowserController().observe(params.tabId, params.maxElements)
|
||||
return { content: [{ type: 'text', text: JSON.stringify(observation) }], details: observation }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_find',
|
||||
label: '语义定位元素',
|
||||
description:
|
||||
'按可访问性 role 和/或 name 定位少量新 ref,不返回整页快照。适合 observe 结果过大或目标被截掉时使用。与 observe 一样会作废同标签页的全部旧 ref。',
|
||||
promptSnippet: 'browser_find: 按 role/name 精准找元素',
|
||||
parameters: Type.Object({
|
||||
role: Type.Optional(Type.String({ description: '可访问性 role,如 button/textbox/link/checkbox' })),
|
||||
name: Type.Optional(Type.String({ description: '可访问性名称查询' })),
|
||||
exact: Type.Optional(Type.Boolean({ description: '精确匹配 name(默认包含匹配)' })),
|
||||
maxResults: Type.Optional(Type.Number({ minimum: 1, maximum: 50, description: '最多返回条数,默认 20' })),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const result = await getBrowserController().find(
|
||||
{ role: params.role, name: params.name, exact: params.exact, maxResults: params.maxResults },
|
||||
params.tabId
|
||||
)
|
||||
return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_click',
|
||||
label: '点击元素',
|
||||
description: '点击最新 observe/find 结果中的指定 ref。页面会短暂高亮目标。点击导致的重渲染会让旧 ref 失效,操作后需重新观察。',
|
||||
promptSnippet: 'browser_click: 点击 ref',
|
||||
parameters: Type.Object({ ref: Type.String({ description: '元素引用(来自 observe/find)' }), tabId: TAB_ID_PARAM }),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().click(params.ref, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ clicked: params.ref, url: state.tabs.find((t) => t.tabId === state.activeTabId)?.url }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_act',
|
||||
label: '点击并等待',
|
||||
description:
|
||||
'点击 ref 后在同一次串行操作中等待一个固定条件(URL 片段/可见文本/CSS selector)。已知点击后的预期状态时优先用它,避免 click+wait 两次调用。',
|
||||
promptSnippet: 'browser_act: 点击 + 等待条件(原子操作)',
|
||||
parameters: Type.Object({
|
||||
ref: Type.String({ description: '元素引用(来自 observe/find)' }),
|
||||
waitFor: Type.Optional(
|
||||
Type.Object({
|
||||
kind: Type.Union([Type.Literal('url'), Type.Literal('text'), Type.Literal('selector')]),
|
||||
value: Type.String({ description: 'URL 片段 / 可见文本 / CSS selector' })
|
||||
})
|
||||
),
|
||||
timeoutMs: Type.Optional(Type.Number({ minimum: 250, maximum: 30000, description: '等待超时毫秒,默认 10000' })),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const result = await getBrowserController().act(params.ref, params.waitFor, params.timeoutMs, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ clicked: params.ref, matched: result.matched }) }], details: result.state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_fill',
|
||||
label: '填写输入框',
|
||||
description:
|
||||
'整体替换指定 ref 的 input/textarea/contenteditable 内容(先全选再插入,支持空格、标点、Unicode 与换行)。完整消息、搜索词、多行文本都优先用它。',
|
||||
promptSnippet: 'browser_fill: 替换输入框内容',
|
||||
parameters: Type.Object({
|
||||
ref: Type.String({ description: '可编辑字段的 ref(observe 结果中 editable=true)' }),
|
||||
text: Type.String({ description: '要输入的完整文本,≤10000 字符' }),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().fill(params.ref, params.text, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ filled: params.ref, chars: Array.from(params.text).length }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_press',
|
||||
label: '按键',
|
||||
description:
|
||||
'按导航键(Enter/Tab/Escape/方向键/PageDown 等)或向已聚焦的编辑器一次性插入完整文本。字段有 ref 时优先 browser_fill。',
|
||||
promptSnippet: 'browser_press: Enter/Tab 等按键或向焦点插入文本',
|
||||
parameters: Type.Object({ key: Type.String({ description: '导航键名,或要插入的完整文本' }), tabId: TAB_ID_PARAM }),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().press(params.key, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ pressed: params.key }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_hover',
|
||||
label: '悬停元素',
|
||||
description: '把指针移到指定 ref 上(真实 mouseMoved 事件),用于展开悬浮菜单/提示,随后需重新观察再点击新内容。',
|
||||
promptSnippet: 'browser_hover: 悬停展开菜单',
|
||||
parameters: Type.Object({ ref: Type.String({ description: '元素引用' }), tabId: TAB_ID_PARAM }),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().hover(params.ref, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ hovered: params.ref }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_scroll',
|
||||
label: '滚动页面',
|
||||
description:
|
||||
'滚动页面或 CSS selector 指定的内部滚动容器(SPA 信息流常在内部容器里,导航键滚不动)。返回前后 scrollTop 指标,用 moved 核验是否真的滚动。',
|
||||
promptSnippet: 'browser_scroll: 滚动页面/内部容器',
|
||||
parameters: Type.Object({
|
||||
deltaY: Type.Optional(Type.Number({ description: '滚动像素,正向下负向上,默认 600' })),
|
||||
position: Type.Optional(Type.Union([Type.Literal('top'), Type.Literal('bottom')], { description: '直接滚到顶部/底部(优先于 deltaY)' })),
|
||||
selector: Type.Optional(Type.String({ description: '内部滚动容器的 CSS selector,缺省滚主文档' })),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const result = await getBrowserController().scroll(
|
||||
{ deltaY: params.deltaY, position: params.position, selector: params.selector },
|
||||
params.tabId
|
||||
)
|
||||
return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_extract',
|
||||
label: '提取页面文本',
|
||||
description:
|
||||
'从页面正文或 selector 区域提取受限长度的纯文本,替代为读内容而执行 JS。优先传 selector 限定正文/列表区域,避免导航栏与页脚噪声。',
|
||||
promptSnippet: 'browser_extract: 提取正文/区域文本',
|
||||
parameters: Type.Object({
|
||||
selector: Type.Optional(Type.String({ description: '限定提取区域的 CSS selector,缺省取 body' })),
|
||||
maxChars: Type.Optional(Type.Number({ description: '最多提取字符数,默认 50000' })),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const result = await getBrowserController().extract({ selector: params.selector, maxChars: params.maxChars }, params.tabId)
|
||||
return { content: [{ type: 'text', text: result.text }], details: { url: result.url, title: result.title, truncated: result.truncated } }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_wait_for',
|
||||
label: '等待页面状态',
|
||||
description: '等待一个固定条件:URL 片段、可见文本或 CSS selector。超时返回 matched=false,不执行任意 JavaScript。',
|
||||
promptSnippet: 'browser_wait_for: 等待 url/text/selector 条件',
|
||||
parameters: Type.Object({
|
||||
kind: Type.Union([Type.Literal('url'), Type.Literal('text'), Type.Literal('selector')]),
|
||||
value: Type.String({ minLength: 1, maxLength: 2000, description: 'URL 片段 / 可见文本 / CSS selector' }),
|
||||
timeoutMs: Type.Optional(Type.Number({ minimum: 250, maximum: 30000, description: '超时毫秒,默认 10000' })),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const result = await getBrowserController().waitFor({ kind: params.kind, value: params.value }, params.timeoutMs, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_screenshot',
|
||||
label: '截图',
|
||||
description: '截取当前页面。语义结构(observe)足够时不要截图;需要视觉验证、布局或渲染证据时使用。',
|
||||
promptSnippet: 'browser_screenshot: 视觉验证用',
|
||||
parameters: Type.Object({ tabId: TAB_ID_PARAM }),
|
||||
async execute(_toolCallId, params) {
|
||||
const shot = await getBrowserController().screenshot(params.tabId)
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'image' as const,
|
||||
data: shot.dataUrl.replace(/^data:image\/\w+;base64,/, ''),
|
||||
mimeType: 'image/png'
|
||||
},
|
||||
{ type: 'text', text: JSON.stringify({ url: shot.url, title: shot.title }) }
|
||||
],
|
||||
details: { url: shot.url, title: shot.title }
|
||||
}
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_list_tabs',
|
||||
label: '列出标签页',
|
||||
description: '列出受管浏览器的全部标签页(tabId/URL/标题/加载状态)与当前激活页。操作其他标签页时先调它拿 tabId。',
|
||||
promptSnippet: 'browser_list_tabs: 查看标签页列表',
|
||||
parameters: Type.Object({}),
|
||||
async execute() {
|
||||
const state = getBrowserController().listTabs()
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ tabs: state.tabs, activeTabId: state.activeTabId }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_new_tab',
|
||||
label: '新建标签页',
|
||||
description: '新建 Agent 工作标签页并激活到用户可见的浏览器面板;完成后不再需要的标签页应主动 browser_close_tab。',
|
||||
promptSnippet: 'browser_new_tab: 新建工作标签页',
|
||||
parameters: Type.Object({
|
||||
url: Type.Optional(Type.String({ description: '初始地址(URL/域名/搜索词),缺省开空白起始页' }))
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().newTab(params.url, true)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ tabId: state.activeTabId, tabs: state.tabs.length }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_select_tab',
|
||||
label: '切换标签页',
|
||||
description: '把指定标签页设为 Agent 工作页,并同步激活到用户可见面板。后续不带 tabId 的操作都作用于它。',
|
||||
promptSnippet: 'browser_select_tab: 切换工作标签页',
|
||||
parameters: Type.Object({ tabId: Type.String({ description: 'browser_list_tabs 返回的标签页 ID' }) }),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().selectTab(params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ activeTabId: state.activeTabId }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_close_tab',
|
||||
label: '关闭标签页',
|
||||
description: '关闭指定标签页(缺省关闭当前工作页)。绝不关闭用户手动打开的标签页。',
|
||||
promptSnippet: 'browser_close_tab: 关闭工作标签页',
|
||||
parameters: Type.Object({ tabId: Type.Optional(Type.String({ description: '要关闭的标签页 ID,缺省当前工作页' })) }),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().closeTab(params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ closed: params.tabId ?? 'current', remaining: state.tabs.length }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_close',
|
||||
label: '关闭浏览器',
|
||||
description: '关闭整个内置受管浏览器会话(销毁全部标签页)。任务结束、不再需要网页时主动调用。',
|
||||
promptSnippet: 'browser_close: 关闭受管浏览器',
|
||||
parameters: Type.Object({}),
|
||||
async execute() {
|
||||
const state = getBrowserController().close()
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ closed: true }) }], details: state }
|
||||
}
|
||||
})
|
||||
]
|
||||
|
||||
export const browserToolNames = new Set(browserTools.map((tool) => tool.name))
|
||||
Reference in New Issue
Block a user