feat: 内置受管浏览器（CDP + AX 快照 ref 体系）+ 本地浏览器配置导入
This commit is contained in:
Rick Ye
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* CDP 命令的基础设施：超时与中止。
|
||||
*
|
||||
* Electron Debugger 的 sendCommand 在目标页面卡死或 DevTools 通道异常时可能永不
|
||||
* settle；超时/中止只让调用方继续执行，底层命令后续 settle 时会被安全忽略。
|
||||
*/
|
||||
|
||||
export const BROWSER_CDP_COMMAND_TIMEOUT_MS = 8_000
|
||||
export const BROWSER_OBSERVE_TIMEOUT_MS = 5_000
|
||||
export const BROWSER_OBSERVE_AX_DEPTH = 8
|
||||
export const BROWSER_OBSERVE_EXTENDED_AX_DEPTH = 16
|
||||
|
||||
/** 高预算 Observe 读取更深的 AX tree，常规观察保持紧凑避免 Chromium 长时间阻塞 */
|
||||
export function resolveBrowserObserveAxDepth(maxElements: number): number {
|
||||
return maxElements > 240 ? BROWSER_OBSERVE_EXTENDED_AX_DEPTH : BROWSER_OBSERVE_AX_DEPTH
|
||||
}
|
||||
|
||||
export class BrowserCdpTimeoutError extends Error {
|
||||
constructor(method: string, timeoutMs: number) {
|
||||
super(`浏览器页面未在 ${Math.ceil(timeoutMs / 1_000)} 秒内响应 ${method}，请稍后重试或重新加载页面。`)
|
||||
this.name = 'BrowserCdpTimeoutError'
|
||||
}
|
||||
}
|
||||
|
||||
/** 已发出的 DevTools 命令无法撤销；该错误只保证不再执行后续页面动作 */
|
||||
export class BrowserOperationAbortedError extends Error {
|
||||
constructor() {
|
||||
super('浏览器操作已停止。已发送的页面指令可能已执行，页面状态请重新观察确认。')
|
||||
this.name = 'BrowserOperationAbortedError'
|
||||
}
|
||||
}
|
||||
|
||||
export function throwIfBrowserOperationAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) throw new BrowserOperationAbortedError()
|
||||
}
|
||||
|
||||
export interface CdpResponse {
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
export function withBrowserCdpTimeout<T>(
|
||||
command: () => Promise<T>,
|
||||
method: string,
|
||||
timeoutMs: number = BROWSER_CDP_COMMAND_TIMEOUT_MS
|
||||
): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
let settled = false
|
||||
const settle = (callback: () => void): void => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
clearTimeout(timer)
|
||||
callback()
|
||||
}
|
||||
const timer = setTimeout(() => settle(() => reject(new BrowserCdpTimeoutError(method, timeoutMs))), timeoutMs)
|
||||
void Promise.resolve()
|
||||
.then(command)
|
||||
.then((value) => settle(() => resolve(value)))
|
||||
.catch((error: unknown) => settle(() => reject(error)))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,939 @@
|
||||
import { app, BrowserWindow, session as electronSession, WebContentsView, type Rectangle, type Session } from 'electron'
|
||||
import { existsSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import type { BrowserImportInfo, BrowserTabInfo, BrowserUiState } from '@shared/types'
|
||||
import {
|
||||
BrowserCdpTimeoutError,
|
||||
BROWSER_OBSERVE_TIMEOUT_MS,
|
||||
resolveBrowserObserveAxDepth,
|
||||
withBrowserCdpTimeout,
|
||||
type CdpResponse
|
||||
} from './browser-cdp'
|
||||
import {
|
||||
assertSafeBrowserUrl,
|
||||
browserObservationNameLimit,
|
||||
NAMELESS_ALLOWED_ROLES,
|
||||
prioritizeBrowserObservationCandidates,
|
||||
resolveBrowserDestination,
|
||||
resolveBrowserObserveMaxElements
|
||||
} from './browser-policy'
|
||||
|
||||
/**
|
||||
* 内置受管浏览器控制器。
|
||||
*
|
||||
* 架构（参考 Proma 的成熟实现，适配本应用单窗口/单聊天会话形态）：
|
||||
* - 每个标签页 = 一个原生 WebContentsView，通过 contentView.addChildView 呈现；
|
||||
* 渲染层 BrowserSlot 量出占位矩形，经 IPC 同步边界（原生视图天然盖在 DOM 之上）。
|
||||
* - 页面观察/操作全部走 CDP（webContents.debugger）：AX tree 快照 + ref 代际机制
|
||||
* + Input.dispatch* 真实输入事件；不在页面里执行模拟点击。
|
||||
* - 登录态隔离在 persist:aso-browser 分区；本地浏览器配置导入由 browser-profile.ts
|
||||
* 直接迁移分区目录数据（Cookies/Local Storage + 密钥移植）。
|
||||
*/
|
||||
|
||||
const PARTITION = 'persist:aso-browser'
|
||||
const NEW_TAB_URL = 'https://www.google.com/'
|
||||
const MAX_TABS = 12
|
||||
const MAX_FILL_CHARS = 10_000
|
||||
const MAX_SCRIPT_CHARS = 20_000
|
||||
const MAX_EXTRACT_CHARS = 50_000
|
||||
|
||||
interface RefEntry {
|
||||
backendNodeId: number
|
||||
generation: number
|
||||
label: string
|
||||
editable: boolean
|
||||
}
|
||||
|
||||
interface TabRecord {
|
||||
tabId: string
|
||||
view: WebContentsView
|
||||
createdAt: number
|
||||
url: string
|
||||
title: string
|
||||
loading: boolean
|
||||
/** ref 表与代际号：每次观察/导航后旧 ref 失效 */
|
||||
refs: Map<string, RefEntry>
|
||||
generation: number
|
||||
/** 同一 tab 的 Agent 操作串行化，避免 CDP 命令交错 */
|
||||
commandTail: Promise<unknown>
|
||||
lastLayoutRevision: number
|
||||
attached: boolean
|
||||
}
|
||||
|
||||
interface BrowserSettings {
|
||||
riskAcknowledged: boolean
|
||||
imported?: BrowserImportInfo | null
|
||||
}
|
||||
|
||||
export interface BrowserObservation {
|
||||
tabId: string
|
||||
url: string
|
||||
title: string
|
||||
generation: number
|
||||
elements: Array<{ ref: string; role: string; name: string; editable: boolean }>
|
||||
}
|
||||
|
||||
export interface BrowserWaitCondition {
|
||||
kind: 'url' | 'text' | 'selector'
|
||||
value: string
|
||||
}
|
||||
|
||||
function textValue(v: unknown): string {
|
||||
if (v && typeof v === 'object') {
|
||||
const value = (v as { value?: unknown }).value
|
||||
if (typeof value === 'string') return value
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function isEditableAxNode(node: Record<string, unknown>): boolean {
|
||||
const role = textValue(node.role).toLowerCase()
|
||||
if (role === 'textfield' || role === 'textbox' || role === 'searchbox') return true
|
||||
const props = Array.isArray(node.properties) ? (node.properties as Array<Record<string, unknown>>) : []
|
||||
for (const p of props) {
|
||||
if (String(p.name) === 'editable') {
|
||||
const inner = p.value as { value?: unknown } | undefined
|
||||
if (inner && typeof inner.value === 'boolean') return inner.value
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** 导航键 → CDP key 事件参数（rawKeyDown + VK code 才会触发默认行为：滚动/提交/焦点移动） */
|
||||
const KEY_EVENTS: Record<string, { key: string; code: string; vk: number }> = {
|
||||
enter: { key: 'Enter', code: 'Enter', vk: 13 },
|
||||
tab: { key: 'Tab', code: 'Tab', vk: 9 },
|
||||
escape: { key: 'Escape', code: 'Escape', vk: 27 },
|
||||
backspace: { key: 'Backspace', code: 'Backspace', vk: 8 },
|
||||
delete: { key: 'Delete', code: 'Delete', vk: 46 },
|
||||
pageup: { key: 'PageUp', code: 'PageUp', vk: 33 },
|
||||
pagedown: { key: 'PageDown', code: 'PageDown', vk: 34 },
|
||||
home: { key: 'Home', code: 'Home', vk: 36 },
|
||||
end: { key: 'End', code: 'End', vk: 35 },
|
||||
arrowup: { key: 'ArrowUp', code: 'ArrowUp', vk: 38 },
|
||||
arrowdown: { key: 'ArrowDown', code: 'ArrowDown', vk: 40 },
|
||||
arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft', vk: 37 },
|
||||
arrowright: { key: 'ArrowRight', code: 'ArrowRight', vk: 39 },
|
||||
space: { key: ' ', code: 'Space', vk: 32 }
|
||||
}
|
||||
|
||||
export class BrowserController {
|
||||
private owner: BrowserWindow | null = null
|
||||
private browserSession: Session
|
||||
private tabs = new Map<string, TabRecord>()
|
||||
private activeTabId: string | null = null
|
||||
private agentTabId: string | null = null
|
||||
private listeners = new Set<(state: BrowserUiState) => void>()
|
||||
private traceText: string | null = null
|
||||
private settings: BrowserSettings = { riskAcknowledged: false }
|
||||
|
||||
constructor() {
|
||||
this.browserSession = electronSession.fromPartition(PARTITION)
|
||||
// 诚实的 UA：保留 Chromium token 保证站点兼容，去掉 Electron 标识
|
||||
this.browserSession.setUserAgent(
|
||||
this.browserSession.getUserAgent().replace(/Electron\/[\d.]+/, `InkreachAgent/${app.getVersion()}`)
|
||||
)
|
||||
// 网页权限请求（摄像头/定位等）默认拒绝
|
||||
this.browserSession.setPermissionRequestHandler((_wc, _permission, callback) => callback(false))
|
||||
// 下载留在受管浏览器内，落到系统下载目录（重名自动加序号）
|
||||
this.browserSession.on('will-download', (_event, item) => {
|
||||
const dir = app.getPath('downloads')
|
||||
const base = item.getFilename() || 'download'
|
||||
const dot = base.lastIndexOf('.')
|
||||
const stem = dot > 0 ? base.slice(0, dot) : base
|
||||
const ext = dot > 0 ? base.slice(dot) : ''
|
||||
let target = join(dir, base)
|
||||
for (let i = 1; existsSync(target) && i < 100; i++) {
|
||||
target = join(dir, `${stem} (${i})${ext}`)
|
||||
}
|
||||
item.setSavePath(target)
|
||||
})
|
||||
this.settings = this.loadSettings()
|
||||
}
|
||||
|
||||
// ---- 设置持久化 ----
|
||||
|
||||
private get settingsFile(): string {
|
||||
return join(app.getPath('userData'), 'aso-browser.json')
|
||||
}
|
||||
|
||||
private loadSettings(): BrowserSettings {
|
||||
try {
|
||||
const raw = JSON.parse(readFileSync(this.settingsFile, 'utf8')) as Partial<BrowserSettings>
|
||||
return {
|
||||
riskAcknowledged: raw.riskAcknowledged === true,
|
||||
imported: raw.imported ?? null
|
||||
}
|
||||
} catch {
|
||||
return { riskAcknowledged: false }
|
||||
}
|
||||
}
|
||||
|
||||
private saveSettings(): void {
|
||||
try {
|
||||
writeFileSync(this.settingsFile, JSON.stringify(this.settings, null, 2), 'utf8')
|
||||
} catch (err) {
|
||||
console.error('[browser] save settings failed:', err)
|
||||
}
|
||||
}
|
||||
|
||||
/** 配置导入完成后由 browser-profile.ts 回写摘要 */
|
||||
setImportedInfo(info: BrowserImportInfo | null): void {
|
||||
this.settings.imported = info
|
||||
this.saveSettings()
|
||||
this.emitState()
|
||||
}
|
||||
|
||||
get importedInfo(): BrowserImportInfo | null {
|
||||
return this.settings.imported ?? null
|
||||
}
|
||||
|
||||
/** 当前是否有活跃标签页（导入配置前必须没有） */
|
||||
get hasTabs(): boolean {
|
||||
return this.tabs.size > 0
|
||||
}
|
||||
|
||||
// ---- 窗口与状态广播 ----
|
||||
|
||||
setOwnerWindow(win: BrowserWindow): void {
|
||||
this.owner = win
|
||||
}
|
||||
|
||||
handleWindowClosed(): void {
|
||||
for (const tab of this.tabs.values()) {
|
||||
try {
|
||||
if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach()
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
try {
|
||||
tab.view.setVisible(false)
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
}
|
||||
this.tabs.clear()
|
||||
this.activeTabId = null
|
||||
this.agentTabId = null
|
||||
this.owner = null
|
||||
}
|
||||
|
||||
onState(fn: (state: BrowserUiState) => void): () => void {
|
||||
this.listeners.add(fn)
|
||||
return () => this.listeners.delete(fn)
|
||||
}
|
||||
|
||||
private buildState(): BrowserUiState {
|
||||
const tabs: BrowserTabInfo[] = [...this.tabs.values()]
|
||||
.sort((a, b) => a.createdAt - b.createdAt)
|
||||
.map((t) => ({ tabId: t.tabId, url: t.url, title: t.title, loading: t.loading }))
|
||||
return {
|
||||
open: tabs.length > 0,
|
||||
riskAcknowledged: this.settings.riskAcknowledged,
|
||||
tabs,
|
||||
activeTabId: this.activeTabId,
|
||||
trace: this.traceText,
|
||||
imported: this.settings.imported ?? null
|
||||
}
|
||||
}
|
||||
|
||||
getState(): BrowserUiState {
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
private emitState(): void {
|
||||
const state = this.buildState()
|
||||
for (const fn of this.listeners) {
|
||||
try {
|
||||
fn(state)
|
||||
} catch (err) {
|
||||
console.error('[browser] state listener error:', err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private trace(tab: TabRecord, action: string, detail: string): void {
|
||||
this.traceText = `[${action}] ${detail}`
|
||||
void tab
|
||||
this.emitState()
|
||||
}
|
||||
|
||||
// ---- 风险告知门禁 ----
|
||||
|
||||
acknowledgeRisk(): void {
|
||||
this.settings.riskAcknowledged = true
|
||||
this.saveSettings()
|
||||
this.emitState()
|
||||
}
|
||||
|
||||
private assertRiskAcknowledged(): void {
|
||||
if (!this.settings.riskAcknowledged) {
|
||||
throw new Error('内置浏览器首次使用尚未确认风险告知：请在应用右侧浏览器面板中阅读并点击「我已了解，开始使用」后重试。')
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 会话与标签页 ----
|
||||
|
||||
private ensureBrowser(): TabRecord {
|
||||
if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪，无法使用内置浏览器。')
|
||||
if (this.tabs.size === 0) this.createTab(NEW_TAB_URL)
|
||||
const active = this.activeTabId ? this.tabs.get(this.activeTabId) : null
|
||||
const fallback = this.tabs.values().next().value
|
||||
const tab = active ?? fallback
|
||||
if (!tab) throw new Error('浏览器标签页不可用。')
|
||||
this.emitState()
|
||||
return tab
|
||||
}
|
||||
|
||||
private invalidateTabDocument(tab: TabRecord): void {
|
||||
tab.generation += 1
|
||||
tab.refs.clear()
|
||||
}
|
||||
|
||||
private createTab(initialUrl: string, viewOptions?: Electron.WebContentsViewConstructorOptions): TabRecord {
|
||||
if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪，无法创建浏览器标签。')
|
||||
if (this.tabs.size >= MAX_TABS) throw new Error(`浏览器标签页已达上限（${MAX_TABS}），请先关闭不需要的标签。`)
|
||||
const tabId = `tab-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
||||
const view = new WebContentsView(
|
||||
viewOptions ?? {
|
||||
webPreferences: {
|
||||
partition: PARTITION,
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
webSecurity: true
|
||||
}
|
||||
}
|
||||
)
|
||||
const tab: TabRecord = {
|
||||
tabId,
|
||||
view,
|
||||
createdAt: Date.now(),
|
||||
url: '',
|
||||
title: '新标签页',
|
||||
loading: false,
|
||||
refs: new Map(),
|
||||
generation: 0,
|
||||
commandTail: Promise.resolve(),
|
||||
lastLayoutRevision: 0,
|
||||
attached: false
|
||||
}
|
||||
view.setVisible(false)
|
||||
|
||||
// popup 收编为受管同级标签（必须同步返回 webContents，故提前建好 TabRecord）
|
||||
view.webContents.setWindowOpenHandler(({ url }) => {
|
||||
try {
|
||||
assertSafeBrowserUrl(url)
|
||||
} catch {
|
||||
this.trace(tab, 'popup', `已阻止不安全的新窗口: ${url.slice(0, 80)}`)
|
||||
return { action: 'deny' }
|
||||
}
|
||||
if (this.tabs.get(tabId) !== tab || this.tabs.size >= MAX_TABS) return { action: 'deny' }
|
||||
return {
|
||||
action: 'allow',
|
||||
outlivesOpener: true,
|
||||
createWindow: (options) => {
|
||||
const popup = this.createTab(url, options)
|
||||
this.selectTab(popup.tabId)
|
||||
this.trace(popup, 'popup', '网页打开了新标签页')
|
||||
return popup.view.webContents
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const updateNav = (): void => {
|
||||
const wc = view.webContents
|
||||
if (wc.isDestroyed()) return
|
||||
try {
|
||||
tab.url = wc.getURL()
|
||||
tab.title = wc.getTitle() || tab.url
|
||||
tab.loading = wc.isLoading()
|
||||
} catch {
|
||||
// 竞态时忽略
|
||||
}
|
||||
this.emitState()
|
||||
}
|
||||
view.webContents.on('did-start-loading', () => {
|
||||
tab.loading = true
|
||||
this.invalidateTabDocument(tab)
|
||||
this.emitState()
|
||||
})
|
||||
view.webContents.on('did-stop-loading', updateNav)
|
||||
view.webContents.on('page-title-updated', updateNav)
|
||||
view.webContents.on('did-navigate', () => {
|
||||
this.invalidateTabDocument(tab)
|
||||
updateNav()
|
||||
})
|
||||
view.webContents.on('did-navigate-in-page', () => {
|
||||
this.invalidateTabDocument(tab)
|
||||
updateNav()
|
||||
})
|
||||
view.webContents.on('destroyed', () => {
|
||||
this.tabs.delete(tabId)
|
||||
this.removeView(view)
|
||||
if (this.activeTabId === tabId) this.activeTabId = this.tabs.keys().next().value ?? null
|
||||
if (this.agentTabId === tabId) this.agentTabId = this.activeTabId
|
||||
this.emitState()
|
||||
})
|
||||
|
||||
try {
|
||||
view.webContents.debugger.attach('1.3')
|
||||
} catch (err) {
|
||||
console.warn('[browser] CDP attach 失败:', err)
|
||||
}
|
||||
this.tabs.set(tabId, tab)
|
||||
if (!this.activeTabId) this.activeTabId = tabId
|
||||
// popup 场景（带 viewOptions）由 Electron 的 createWindow 机制自行导航，避免双重加载
|
||||
if (initialUrl && !viewOptions) void view.webContents.loadURL(assertSafeBrowserUrl(initialUrl)).catch(() => undefined)
|
||||
return tab
|
||||
}
|
||||
|
||||
private removeView(view: WebContentsView): void {
|
||||
try {
|
||||
this.owner?.contentView.removeChildView(view)
|
||||
} catch {
|
||||
// 窗口已销毁
|
||||
}
|
||||
try {
|
||||
view.webContents.close()
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
}
|
||||
|
||||
listTabs(): BrowserUiState {
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
newTab(url?: string, byAgent = true): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.createTab(url ? resolveBrowserDestination(url) : NEW_TAB_URL)
|
||||
this.activeTabId = tab.tabId
|
||||
if (byAgent) this.agentTabId = tab.tabId
|
||||
this.emitState()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
selectTab(tabId: string): BrowserUiState {
|
||||
const tab = this.tabs.get(tabId)
|
||||
if (!tab) throw new Error(`标签页不存在: ${tabId}`)
|
||||
this.activeTabId = tabId
|
||||
try {
|
||||
tab.view.webContents.focus()
|
||||
} catch {
|
||||
// 页面可能正在加载
|
||||
}
|
||||
this.emitState()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
closeTab(tabId?: string): BrowserUiState {
|
||||
const id = tabId ?? this.activeTabId
|
||||
if (!id) return this.buildState()
|
||||
const tab = this.tabs.get(id)
|
||||
if (!tab) return this.buildState()
|
||||
try {
|
||||
if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach()
|
||||
} catch {
|
||||
// 已销毁
|
||||
}
|
||||
this.removeView(tab.view)
|
||||
this.tabs.delete(id)
|
||||
if (this.activeTabId === id) this.activeTabId = this.tabs.keys().next().value ?? null
|
||||
if (this.agentTabId === id) this.agentTabId = this.activeTabId
|
||||
this.emitState()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
/** 关闭整个受管浏览器（销毁全部标签；导入配置前也会走这里） */
|
||||
close(): BrowserUiState {
|
||||
for (const tabId of [...this.tabs.keys()]) this.closeTab(tabId)
|
||||
this.agentTabId = null
|
||||
this.traceText = null
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
// ---- 渲染层布局同步 ----
|
||||
|
||||
applyLayout(layout: { tabId: string; visible: boolean; bounds: Rectangle; revision: number }): void {
|
||||
const tab = this.tabs.get(layout.tabId)
|
||||
if (!tab || !this.owner || this.owner.isDestroyed()) return
|
||||
if (layout.revision <= tab.lastLayoutRevision) return
|
||||
tab.lastLayoutRevision = layout.revision
|
||||
try {
|
||||
if (layout.visible && layout.bounds.width > 4 && layout.bounds.height > 4) {
|
||||
if (!tab.attached) {
|
||||
this.owner.contentView.addChildView(tab.view)
|
||||
tab.attached = true
|
||||
}
|
||||
tab.view.setBounds(layout.bounds)
|
||||
tab.view.setVisible(true)
|
||||
} else {
|
||||
tab.view.setVisible(false)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[browser] apply layout failed:', err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 用户导航（地址栏/按钮） ----
|
||||
|
||||
navigateDisplay(input: string): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
void tab.view.webContents.loadURL(resolveBrowserDestination(input)).catch(() => undefined)
|
||||
this.trace(tab, 'navigate', `打开 ${input.slice(0, 80)}`)
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
goBack(): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
if (tab.view.webContents.navigationHistory.canGoBack()) tab.view.webContents.navigationHistory.goBack()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
goForward(): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
if (tab.view.webContents.navigationHistory.canGoForward()) tab.view.webContents.navigationHistory.goForward()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
reload(): BrowserUiState {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.ensureBrowser()
|
||||
tab.view.webContents.reload()
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
// ---- CDP 基础 ----
|
||||
|
||||
private cdp(tab: TabRecord, method: string, params?: Record<string, unknown>, timeoutMs?: number): Promise<CdpResponse> {
|
||||
const debuggerClient = tab.view.webContents.debugger
|
||||
if (!debuggerClient.isAttached()) throw new Error('浏览器调试通道不可用，请重新打开该标签页。')
|
||||
return withBrowserCdpTimeout(() => debuggerClient.sendCommand(method, params) as Promise<CdpResponse>, method, timeoutMs).catch(
|
||||
(error: unknown) => {
|
||||
if (error instanceof BrowserCdpTimeoutError) this.recoverDebugger(tab, method)
|
||||
throw error
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/** 超时后重连 debugger，避免一个卡死页面让后续浏览器调用全部失败 */
|
||||
private recoverDebugger(tab: TabRecord, method: string): void {
|
||||
try {
|
||||
const debuggerClient = tab.view.webContents.debugger
|
||||
if (debuggerClient.isAttached()) debuggerClient.detach()
|
||||
debuggerClient.attach('1.3')
|
||||
console.warn(`[browser] ${method} 超时，已重连调试通道`)
|
||||
} catch {
|
||||
// 页面已关闭
|
||||
}
|
||||
}
|
||||
|
||||
/** Agent 的 tab 内操作串行化（用户操作不排队） */
|
||||
private runAgentOp<T>(tab: TabRecord, fn: () => Promise<T>): Promise<T> {
|
||||
const next = tab.commandTail.then(fn, fn)
|
||||
tab.commandTail = next.catch(() => undefined)
|
||||
return next
|
||||
}
|
||||
|
||||
private getAgentTab(tabId?: string): TabRecord {
|
||||
if (tabId) {
|
||||
const explicit = this.tabs.get(tabId)
|
||||
if (!explicit) throw new Error('目标标签页不存在，请先 browser_list_tabs 并用 browser_new_tab/browser_select_tab 选择。')
|
||||
return explicit
|
||||
}
|
||||
const id = this.agentTabId ?? this.activeTabId
|
||||
const tab = id ? this.tabs.get(id) : null
|
||||
if (tab) return tab
|
||||
// Agent 首次操作时还没有任何标签页：自动开一个工作页
|
||||
return this.ensureBrowser()
|
||||
}
|
||||
|
||||
private assertCurrentDocument(tab: TabRecord, generation: number): void {
|
||||
if (tab.generation !== generation) {
|
||||
throw new Error('页面已变化（导航或重渲染），旧元素引用失效：请重新 browser_observe 后再操作。')
|
||||
}
|
||||
}
|
||||
|
||||
private resolveRef(tab: TabRecord, ref: string): RefEntry {
|
||||
const entry = tab.refs.get(ref)
|
||||
if (!entry || entry.generation !== tab.generation) {
|
||||
throw new Error('元素引用已失效，请先重新调用 browser_observe。')
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
/** AX ref → 视口中心坐标（先滚动到可见再读 box；懒加载列表滚动后坐标会变，不能复用旧值） */
|
||||
private async centerForRef(tab: TabRecord, ref: string, generation: number): Promise<{ x: number; y: number }> {
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
const { backendNodeId } = this.resolveRef(tab, ref)
|
||||
await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
const box = await this.cdp(tab, 'DOM.getBoxModel', { backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
const model = box.model as Record<string, unknown> | undefined
|
||||
const quad = Array.isArray(model?.content) ? (model?.content as unknown[]) : []
|
||||
if (quad.length < 8 || !quad.every((v) => typeof v === 'number')) {
|
||||
throw new Error('目标元素当前不可点击，请重新观察页面。')
|
||||
}
|
||||
const n = quad as number[]
|
||||
return {
|
||||
x: (n[0] + n[2] + n[4] + n[6]) / 4,
|
||||
y: (n[1] + n[3] + n[5] + n[7]) / 4
|
||||
}
|
||||
}
|
||||
|
||||
/** 点击前在页面上短暂高亮目标，用户能看清 Agent 正在操作什么 */
|
||||
private async highlightAgentTarget(tab: TabRecord, backendNodeId: number): Promise<void> {
|
||||
try {
|
||||
const resolved = await this.cdp(tab, 'DOM.resolveNode', { backendNodeId })
|
||||
const objectId = (resolved.object as { objectId?: string } | undefined)?.objectId
|
||||
if (!objectId) return
|
||||
await this.cdp(tab, 'Runtime.callFunctionOn', {
|
||||
functionDeclaration:
|
||||
'function(){const r=this.getBoundingClientRect();const d=document.createElement("div");' +
|
||||
'd.style.cssText=`position:fixed;left:${r.left-2}px;top:${r.top-2}px;width:${r.width+4}px;height:${r.height+4}px;`+' +
|
||||
'"outline:2px solid #f59e0b;outline-offset:1px;pointer-events:none;z-index:2147483647;";' +
|
||||
'document.documentElement.appendChild(d);setTimeout(()=>d.remove(),700);return true}',
|
||||
objectId,
|
||||
returnByValue: true
|
||||
})
|
||||
} catch {
|
||||
// 高亮失败不影响操作
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Agent 工具实现 ----
|
||||
|
||||
async navigate(url: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
this.agentTabId = tab.tabId
|
||||
this.activeTabId = tab.tabId
|
||||
const destination = resolveBrowserDestination(url)
|
||||
await tab.view.webContents.loadURL(destination)
|
||||
this.trace(tab, 'navigate', `打开 ${destination.slice(0, 80)}`)
|
||||
return this.buildState()
|
||||
}
|
||||
|
||||
private parseAxNodes(response: CdpResponse): Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> {
|
||||
const nodes = Array.isArray(response.nodes) ? (response.nodes as Array<Record<string, unknown>>) : []
|
||||
const candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> = []
|
||||
for (const node of nodes) {
|
||||
if (!node) continue
|
||||
const backendNodeId = typeof node.backendDOMNodeId === 'number' ? node.backendDOMNodeId : 0
|
||||
const role = textValue(node.role).toLowerCase()
|
||||
const name = textValue(node.name)
|
||||
const editable = isEditableAxNode(node)
|
||||
if (!backendNodeId || !role) continue
|
||||
if (!name && !editable && !NAMELESS_ALLOWED_ROLES.has(role)) continue
|
||||
candidates.push({ backendNodeId, role, name: name.slice(0, browserObservationNameLimit(role)), editable })
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
private buildRefs(tab: TabRecord, candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }>): BrowserObservation['elements'] {
|
||||
tab.generation += 1
|
||||
tab.refs.clear()
|
||||
const elements: BrowserObservation['elements'] = []
|
||||
for (const candidate of candidates) {
|
||||
const ref = `r${tab.generation}-${elements.length + 1}`
|
||||
tab.refs.set(ref, {
|
||||
backendNodeId: candidate.backendNodeId,
|
||||
generation: tab.generation,
|
||||
label: candidate.name ? `${candidate.role}「${candidate.name.slice(0, 80)}」` : candidate.role,
|
||||
editable: candidate.editable
|
||||
})
|
||||
elements.push({ ref, role: candidate.role, name: candidate.name, editable: candidate.editable })
|
||||
}
|
||||
return elements
|
||||
}
|
||||
|
||||
async observe(tabId?: string, requestedMaxElements?: number): Promise<BrowserObservation> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
this.agentTabId = tab.tabId
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const maxElements = resolveBrowserObserveMaxElements(requestedMaxElements)
|
||||
const depth = resolveBrowserObserveAxDepth(maxElements)
|
||||
const response = await this.cdp(
|
||||
tab,
|
||||
'Accessibility.getFullAXTree',
|
||||
{ depth },
|
||||
BROWSER_OBSERVE_TIMEOUT_MS
|
||||
)
|
||||
const selected = prioritizeBrowserObservationCandidates(this.parseAxNodes(response), maxElements)
|
||||
const elements = this.buildRefs(tab, selected)
|
||||
this.trace(tab, 'observe', `读取到 ${elements.length}/${maxElements} 个元素（AX 深度 ${depth}）`)
|
||||
return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements }
|
||||
})
|
||||
}
|
||||
|
||||
async find(
|
||||
query: { role?: string; name?: string; exact?: boolean; maxResults?: number },
|
||||
tabId?: string
|
||||
): Promise<BrowserObservation> {
|
||||
const role = query.role?.trim().toLowerCase()
|
||||
const name = query.name?.trim().toLowerCase()
|
||||
const maxResults = query.maxResults ?? 20
|
||||
if (!role && !name) throw new Error('语义定位至少需要 role 或 name。')
|
||||
if (!Number.isInteger(maxResults) || maxResults < 1 || maxResults > 50) throw new Error('maxResults 必须是 1 到 50 的整数。')
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
this.agentTabId = tab.tabId
|
||||
return this.runAgentOp(tab, async () => {
|
||||
// 过滤后再套观察配额：Find 能找到被紧凑 Observe 截掉的目标
|
||||
const response = await this.cdp(
|
||||
tab,
|
||||
'Accessibility.getFullAXTree',
|
||||
{ depth: resolveBrowserObserveAxDepth(MAX_BROWSER_OBSERVE_DEPTH_FALLBACK) },
|
||||
BROWSER_OBSERVE_TIMEOUT_MS
|
||||
)
|
||||
const candidates = this.parseAxNodes(response).filter((candidate) => {
|
||||
const roleMatches = !role || candidate.role === role
|
||||
const candidateName = candidate.name.toLowerCase()
|
||||
const nameMatches = !name || (query.exact ? candidateName === name : candidateName.includes(name))
|
||||
return roleMatches && nameMatches
|
||||
})
|
||||
const elements = this.buildRefs(tab, candidates.slice(0, maxResults))
|
||||
this.trace(tab, 'find', `语义定位到 ${elements.length} 个元素（${role ?? name}）`)
|
||||
return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements }
|
||||
})
|
||||
}
|
||||
|
||||
private async clickRef(tab: TabRecord, ref: string, generation: number): Promise<RefEntry> {
|
||||
const target = this.resolveRef(tab, ref)
|
||||
const { x, y } = await this.centerForRef(tab, ref, generation)
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 })
|
||||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 })
|
||||
return target
|
||||
}
|
||||
|
||||
async click(ref: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = await this.clickRef(tab, ref, generation)
|
||||
this.trace(tab, 'click', `点击 ${target.label}`)
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
/** 点击 + 等待合并为一次串行操作（已知点击后预期状态时优先使用） */
|
||||
async act(ref: string, waitFor: BrowserWaitCondition | undefined, timeoutMs = 10_000, tabId?: string): Promise<{ state: BrowserUiState; matched: boolean }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = await this.clickRef(tab, ref, generation)
|
||||
this.trace(tab, 'click', `点击 ${target.label}`)
|
||||
if (!waitFor) return { state: this.buildState(), matched: true }
|
||||
const matched = await this.waitForInternal(tab, waitFor, timeoutMs)
|
||||
return { state: this.buildState(), matched }
|
||||
})
|
||||
}
|
||||
|
||||
async hover(ref: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = this.resolveRef(tab, ref)
|
||||
const { x, y } = await this.centerForRef(tab, ref, generation)
|
||||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y })
|
||||
this.trace(tab, 'hover', `悬停 ${target.label}`)
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
async fill(ref: string, text: string, tabId?: string): Promise<BrowserUiState> {
|
||||
if (text.length > MAX_FILL_CHARS) throw new Error(`单次输入不能超过 ${MAX_FILL_CHARS} 个字符。`)
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const generation = tab.generation
|
||||
const target = this.resolveRef(tab, ref)
|
||||
if (!target.editable) throw new Error('目标元素不是可编辑字段，请重新观察后选择 input、textarea 或 contenteditable。')
|
||||
await this.highlightAgentTarget(tab, target.backendNodeId)
|
||||
await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.cdp(tab, 'DOM.focus', { backendNodeId: target.backendNodeId })
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
// 全选（Ctrl/Cmd+A）后 insertText 整段替换，不逐字符模拟
|
||||
const selectAllModifier = process.platform === 'darwin' ? 4 : 2
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', {
|
||||
type: 'keyDown',
|
||||
key: 'a',
|
||||
code: 'KeyA',
|
||||
windowsVirtualKeyCode: 65,
|
||||
modifiers: selectAllModifier
|
||||
})
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', {
|
||||
type: 'keyUp',
|
||||
key: 'a',
|
||||
code: 'KeyA',
|
||||
windowsVirtualKeyCode: 65,
|
||||
modifiers: selectAllModifier
|
||||
})
|
||||
this.assertCurrentDocument(tab, generation)
|
||||
await this.cdp(tab, 'Input.insertText', { text })
|
||||
this.trace(tab, 'fill', `在 ${target.label} 输入 ${Array.from(text).length} 个字符`)
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
async press(key: string, tabId?: string): Promise<BrowserUiState> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const normalized = key.trim().toLowerCase().replace(/\s+/g, '')
|
||||
const keyEvent = KEY_EVENTS[normalized]
|
||||
if (keyEvent) {
|
||||
const event = { key: keyEvent.key, code: keyEvent.code, windowsVirtualKeyCode: keyEvent.vk }
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'rawKeyDown', ...event })
|
||||
await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'keyUp', ...event })
|
||||
this.trace(tab, 'press', `按下 ${keyEvent.key}`)
|
||||
} else {
|
||||
await this.cdp(tab, 'Input.insertText', { text: key })
|
||||
this.trace(tab, 'press', `输入 ${Array.from(key).length} 个字符`)
|
||||
}
|
||||
return this.buildState()
|
||||
})
|
||||
}
|
||||
|
||||
async scroll(
|
||||
input: { deltaY?: number; position?: 'top' | 'bottom'; selector?: string },
|
||||
tabId?: string
|
||||
): Promise<{ before: { top: number; height: number }; after: { top: number; height: number }; moved: boolean }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : ''
|
||||
const position = input.position === 'top' || input.position === 'bottom' ? input.position : undefined
|
||||
const deltaY = Math.max(-50_000, Math.min(50_000, Math.round(input.deltaY ?? 600)))
|
||||
// 固定表达式：数据由 JSON 注入，不拼接用户脚本
|
||||
const expression =
|
||||
'((input)=>{const el=input.selector?document.querySelector(input.selector):(document.scrollingElement||document.documentElement);' +
|
||||
'if(!el)return {ok:false};const read=()=>({top:Math.round(el.scrollTop),height:Math.round(el.scrollHeight)});' +
|
||||
'const before=read();const top=input.position==="top"?0:input.position==="bottom"?Math.max(0,el.scrollHeight-el.clientHeight):el.scrollTop+input.deltaY;' +
|
||||
'el.scrollTo({top});const after=read();return {ok:true,before,after}})(' +
|
||||
JSON.stringify({ selector, position, deltaY }) +
|
||||
')'
|
||||
const response = await this.evaluateOnPage(tab, expression)
|
||||
const result = (response ?? {}) as { ok?: boolean; before?: { top: number; height: number }; after?: { top: number; height: number } }
|
||||
if (!result.ok || !result.before || !result.after) throw new Error('滚动目标不存在或不支持滚动，请检查 selector。')
|
||||
this.trace(tab, 'scroll', `滚动页面${selector ? `（${selector.slice(0, 40)}）` : ''}`)
|
||||
return { before: result.before, after: result.after, moved: result.after.top !== result.before.top }
|
||||
})
|
||||
}
|
||||
|
||||
async extract(
|
||||
input: { selector?: string; maxChars?: number },
|
||||
tabId?: string
|
||||
): Promise<{ url: string; title: string; text: string; truncated: boolean }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : ''
|
||||
const maxChars = Math.min(MAX_EXTRACT_CHARS, Math.max(500, Math.round(input.maxChars ?? MAX_EXTRACT_CHARS)))
|
||||
const expression =
|
||||
'((input)=>{const root=input.selector?document.querySelector(input.selector):document.body;' +
|
||||
'if(!root)return null;const text=(root.innerText||"").replace(/\\n{3,}/g,"\\n\\n");' +
|
||||
'return {text:text.slice(0,input.maxChars),truncated:text.length>input.maxChars}})(' +
|
||||
JSON.stringify({ selector, maxChars }) +
|
||||
')'
|
||||
const response = await this.evaluateOnPage(tab, expression)
|
||||
const result = (response ?? {}) as { text?: string; truncated?: boolean } | null
|
||||
if (!result || typeof result.text !== 'string') throw new Error('提取目标不存在，请检查 selector。')
|
||||
this.trace(tab, 'extract', `提取正文 ${result.text.length} 字符${selector ? `（${selector.slice(0, 40)}）` : ''}`)
|
||||
return { url: tab.url, title: tab.title, text: result.text, truncated: result.truncated === true }
|
||||
})
|
||||
}
|
||||
|
||||
async waitFor(condition: BrowserWaitCondition, timeoutMs = 10_000, tabId?: string): Promise<{ matched: boolean; url: string; title: string }> {
|
||||
if (!condition.value.trim()) throw new Error('等待条件不能为空。')
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs < 250 || timeoutMs > 30_000) throw new Error('等待超时必须在 250 到 30000 毫秒之间。')
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const matched = await this.waitForInternal(tab, condition, timeoutMs)
|
||||
this.trace(tab, 'wait', `${matched ? '已满足' : '超时未满足'}${condition.kind} 条件（${condition.value.slice(0, 40)}）`)
|
||||
return { matched, url: tab.url, title: tab.title }
|
||||
})
|
||||
}
|
||||
|
||||
/** 轮询固定表达式（只读检查，不执行任意 JS） */
|
||||
private async waitForInternal(tab: TabRecord, condition: BrowserWaitCondition, timeoutMs: number): Promise<boolean> {
|
||||
const startedAt = Date.now()
|
||||
const payload = JSON.stringify(condition).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029')
|
||||
const expression =
|
||||
`(() => { const condition = ${payload}; try { ` +
|
||||
"if (condition.kind === 'url') return location.href.includes(condition.value); " +
|
||||
"if (condition.kind === 'text') return (document.body?.innerText || '').includes(condition.value); " +
|
||||
'return !!document.querySelector(condition.value); } catch { return false; } })()'
|
||||
while (Date.now() - startedAt <= timeoutMs) {
|
||||
const result = await this.evaluateOnPage(tab, expression)
|
||||
if (result === true) return true
|
||||
await new Promise((resolve) => setTimeout(resolve, 250))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** Agent 逃生舱：在页面上下文执行自写 JS（有长度上限，结果 JSON 化） */
|
||||
async evaluate(script: string, tabId?: string): Promise<unknown> {
|
||||
if (!script.trim()) throw new Error('脚本不能为空。')
|
||||
if (script.length > MAX_SCRIPT_CHARS) throw new Error(`脚本不能超过 ${MAX_SCRIPT_CHARS} 字符。`)
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const result = await this.evaluateOnPage(tab, script, true)
|
||||
this.trace(tab, 'js', `执行页面脚本 ${script.length} 字符`)
|
||||
return result
|
||||
})
|
||||
}
|
||||
|
||||
private async evaluateOnPage(tab: TabRecord, expression: string, userGesture = false): Promise<unknown> {
|
||||
const response = await this.cdp(tab, 'Runtime.evaluate', {
|
||||
expression,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
userGesture
|
||||
})
|
||||
const exception = response.exceptionDetails as { text?: string; exception?: { description?: string } } | undefined
|
||||
if (exception) {
|
||||
throw new Error(`页面脚本执行出错: ${exception.exception?.description || exception.text || '未知错误'}`)
|
||||
}
|
||||
const result = response.result as { type?: string; value?: unknown; description?: string } | undefined
|
||||
if (!result) return null
|
||||
if ('value' in result) return result.value ?? null
|
||||
return { type: result.type ?? 'unknown', description: result.description ?? null }
|
||||
}
|
||||
|
||||
async screenshot(tabId?: string): Promise<{ tabId: string; url: string; title: string; dataUrl: string }> {
|
||||
this.assertRiskAcknowledged()
|
||||
const tab = this.getAgentTab(tabId)
|
||||
return this.runAgentOp(tab, async () => {
|
||||
const image = await tab.view.webContents.capturePage()
|
||||
this.trace(tab, 'screenshot', '截取当前页面')
|
||||
return { tabId: tab.tabId, url: tab.url, title: tab.title, dataUrl: image.toDataURL() }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_BROWSER_OBSERVE_DEPTH_FALLBACK = 400
|
||||
|
||||
let controllerInstance: BrowserController | null = null
|
||||
|
||||
/** 惰性单例：首次取用时才创建（session 相关 API 需在 app ready 后调用） */
|
||||
export function getBrowserController(): BrowserController {
|
||||
if (!controllerInstance) controllerInstance = new BrowserController()
|
||||
return controllerInstance
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* 受管浏览器的 URL 与观察策略。
|
||||
* 独立成模块便于单测与复用（渲染层不需要，主进程专用）。
|
||||
*/
|
||||
|
||||
const SEARCH_URL = 'https://www.google.com/search?q='
|
||||
|
||||
/** 导航白名单：普通 web、本地开发服务与 about:blank；file:/javascript:/data: 一律拒绝 */
|
||||
export function assertSafeBrowserUrl(input: string): string {
|
||||
const value = input.trim()
|
||||
if (!value) throw new Error('URL 不能为空。')
|
||||
if (value === 'about:blank') return value
|
||||
let parsed: URL
|
||||
try {
|
||||
parsed = new URL(value)
|
||||
} catch {
|
||||
throw new Error(`无法解析的 URL: ${value}`)
|
||||
}
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
||||
throw new Error(`不支持的地址协议（仅允许 http/https 与 about:blank）: ${parsed.protocol}`)
|
||||
}
|
||||
return parsed.toString()
|
||||
}
|
||||
|
||||
function isLikelyUrl(value: string): boolean {
|
||||
const candidate = `http://${value}`
|
||||
try {
|
||||
const url = new URL(candidate)
|
||||
const host = url.hostname.toLowerCase()
|
||||
// 域名 / IP（v4/v6）/ localhost 与 *.localhost 视为直达地址
|
||||
const isHost =
|
||||
host === 'localhost' ||
|
||||
host.endsWith('.localhost') ||
|
||||
host.includes('.') ||
|
||||
host.includes(':')
|
||||
// 无空格且形如域名/IP/localhost
|
||||
return isHost && !/\s/.test(value)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** 输入归一化：URL/域名/IP/localhost 直达，普通文本走搜索 */
|
||||
export function resolveBrowserDestination(input: string): string {
|
||||
const value = input.trim()
|
||||
if (!value) throw new Error('地址不能为空。')
|
||||
if (value === 'about:blank') return value
|
||||
if (/^https?:\/\//i.test(value)) return assertSafeBrowserUrl(value)
|
||||
if (isLikelyUrl(value)) return assertSafeBrowserUrl(`https://${value}`)
|
||||
return SEARCH_URL + encodeURIComponent(value)
|
||||
}
|
||||
|
||||
// ---- Observe 观察策略 ----
|
||||
|
||||
export const DEFAULT_BROWSER_OBSERVE_MAX_ELEMENTS = 240
|
||||
export const MAX_BROWSER_OBSERVE_MAX_ELEMENTS = 400
|
||||
export const MIN_BROWSER_OBSERVE_MAX_ELEMENTS = 20
|
||||
|
||||
/** 可交互 AX role 白名单（占观察结果前 2/3 配额） */
|
||||
export const INTERACTIVE_AX_ROLES = new Set([
|
||||
'button',
|
||||
'link',
|
||||
'textfield',
|
||||
'textbox',
|
||||
'searchbox',
|
||||
'checkbox',
|
||||
'radiobutton',
|
||||
'combobox',
|
||||
'menuitem',
|
||||
'menuitemcheckbox',
|
||||
'menuitemradio',
|
||||
'tab',
|
||||
'option',
|
||||
'switch',
|
||||
'slider'
|
||||
])
|
||||
|
||||
/** 无名称也值得返回的 role（依赖 role 本身可定位） */
|
||||
export const NAMELESS_ALLOWED_ROLES = new Set(['button', 'textfield', 'textbox', 'link', 'checkbox', 'combobox'])
|
||||
|
||||
export function resolveBrowserObserveMaxElements(requested?: number): number {
|
||||
if (typeof requested !== 'number' || !Number.isFinite(requested)) return DEFAULT_BROWSER_OBSERVE_MAX_ELEMENTS
|
||||
return Math.min(MAX_BROWSER_OBSERVE_MAX_ELEMENTS, Math.max(MIN_BROWSER_OBSERVE_MAX_ELEMENTS, Math.round(requested)))
|
||||
}
|
||||
|
||||
export function isInteractiveAxRole(role: string): boolean {
|
||||
return INTERACTIVE_AX_ROLES.has(role.toLowerCase())
|
||||
}
|
||||
|
||||
export interface BrowserObservationCandidate {
|
||||
role: string
|
||||
name: string
|
||||
editable?: boolean
|
||||
}
|
||||
|
||||
/** 可交互元素优先（约 2/3 配额），其余语义上下文补足 */
|
||||
export function prioritizeBrowserObservationCandidates<T extends BrowserObservationCandidate>(
|
||||
candidates: readonly T[],
|
||||
maxElements: number
|
||||
): T[] {
|
||||
const interactiveLimit = Math.ceil(maxElements * (2 / 3))
|
||||
const interactive: T[] = []
|
||||
const context: T[] = []
|
||||
for (const candidate of candidates) {
|
||||
if (interactive.length < interactiveLimit && (isInteractiveAxRole(candidate.role) || candidate.editable)) {
|
||||
interactive.push(candidate)
|
||||
} else {
|
||||
context.push(candidate)
|
||||
}
|
||||
}
|
||||
const contextLimit = maxElements - interactive.length
|
||||
return [...interactive, ...context.slice(0, Math.max(0, contextLimit))]
|
||||
}
|
||||
|
||||
/** 不同 role 的可访问名称截断上限，避免长文本撑爆快照 */
|
||||
export function browserObservationNameLimit(role: string): number {
|
||||
if (role === 'link' || role === 'button' || role === 'menuitem') return 80
|
||||
if (role === 'heading') return 120
|
||||
return 160
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
import { app } from 'electron'
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
||||
import { homedir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import type { BrowserImportResult, BrowserProfileMeta, BrowserSource } from '@shared/types'
|
||||
import { getBrowserController } from './browser-controller'
|
||||
|
||||
/**
|
||||
* 本地浏览器配置导入（复用各网站登录态）。
|
||||
*
|
||||
* 实现思路（零解密依赖，纯文件迁移）：
|
||||
* - Local Storage / IndexedDB 在 Chromium 里是明文 leveldb，直接拷入受管分区即可生效。
|
||||
* - Cookies 用 AES-GCM 加密，密钥经 OS 加密后存于浏览器根目录 Local State
|
||||
* （os_crypt.encrypted_key）。Windows 上该密钥由 DPAPI（用户级）包裹——同一
|
||||
* Windows 用户的任意进程都能解开，因此把这份密钥「移植」进本应用的 Local State，
|
||||
* 本应用内的 Chromium 即可解密拷贝过来的 Cookies（Chrome 127+ 的 app-bound
|
||||
* 加密 V20 除外，会明确提示不支持）。macOS/Linux 的密钥绑定钥匙串/KWallet，
|
||||
* Cookies 无法直接迁移，仅迁移 Local Storage。
|
||||
*/
|
||||
|
||||
/** Chromium 系浏览器的用户数据根目录（按平台） */
|
||||
const SOURCE_ROOTS: Array<{ id: string; name: string; win: string; mac: string; linux: string }> = [
|
||||
{
|
||||
id: 'chrome',
|
||||
name: 'Google Chrome',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'Google', 'Chrome', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'Google', 'Chrome'),
|
||||
linux: join(homedir(), '.config', 'google-chrome')
|
||||
},
|
||||
{
|
||||
id: 'edge',
|
||||
name: 'Microsoft Edge',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'Microsoft', 'Edge', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'Microsoft Edge'),
|
||||
linux: join(homedir(), '.config', 'microsoft-edge')
|
||||
},
|
||||
{
|
||||
id: 'brave',
|
||||
name: 'Brave',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'BraveSoftware', 'Brave-Browser', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'BraveSoftware', 'Brave-Browser'),
|
||||
linux: join(homedir(), '.config', 'BraveSoftware', 'Brave-Browser')
|
||||
},
|
||||
{
|
||||
id: 'chromium',
|
||||
name: 'Chromium',
|
||||
win: join(process.env.LOCALAPPDATA ?? '', 'Chromium', 'User Data'),
|
||||
mac: join(homedir(), 'Library', 'Application Support', 'Chromium'),
|
||||
linux: join(homedir(), '.config', 'chromium')
|
||||
}
|
||||
]
|
||||
|
||||
/** 受管浏览器分区目录（persist:aso-browser 的落盘位置） */
|
||||
function partitionDir(): string {
|
||||
return join(app.getPath('userData'), 'Partitions', 'aso-browser')
|
||||
}
|
||||
|
||||
function sourceRoot(id: string): string {
|
||||
const source = SOURCE_ROOTS.find((s) => s.id === id)
|
||||
if (!source) return ''
|
||||
return process.platform === 'win32' ? source.win : process.platform === 'darwin' ? source.mac : source.linux
|
||||
}
|
||||
|
||||
/** 读取 profile 的展示名（Preferences 的 profile.name，失败时用目录名） */
|
||||
function readProfileName(profileDir: string): string {
|
||||
try {
|
||||
const file = join(profileDir, 'Preferences')
|
||||
if (statSync(file).size > 8 * 1024 * 1024) return profileDir.split(/[\\/]/).pop() || profileDir
|
||||
const raw = JSON.parse(readFileSync(file, 'utf8')) as { profile?: { name?: string } }
|
||||
return raw.profile?.name?.trim() || profileDir.split(/[\\/]/).pop() || profileDir
|
||||
} catch {
|
||||
return profileDir.split(/[\\/]/).pop() || profileDir
|
||||
}
|
||||
}
|
||||
|
||||
/** 检测本机可导入的浏览器与 profile 列表 */
|
||||
export function detectBrowserSources(): BrowserSource[] {
|
||||
const sources: BrowserSource[] = []
|
||||
for (const source of SOURCE_ROOTS) {
|
||||
const root = sourceRoot(source.id)
|
||||
if (!root || !existsSync(root)) continue
|
||||
const profiles: BrowserProfileMeta[] = []
|
||||
try {
|
||||
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue
|
||||
if (entry.name.startsWith('.') || entry.name === 'System Profile' || entry.name === 'Guest Profile') continue
|
||||
const profileDir = join(root, entry.name)
|
||||
if (!existsSync(join(profileDir, 'Preferences'))) continue
|
||||
profiles.push({ dir: entry.name, name: readProfileName(profileDir) })
|
||||
}
|
||||
} catch {
|
||||
// 目录不可读则跳过该浏览器
|
||||
continue
|
||||
}
|
||||
if (profiles.length > 0) {
|
||||
sources.push({ id: source.id, name: source.name, userDataDir: root, profiles })
|
||||
}
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
function copyFileSafe(from: string, to: string): void {
|
||||
try {
|
||||
copyFileSync(from, to)
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code
|
||||
if (code === 'EBUSY' || code === 'EPERM' || code === 'EACCES') {
|
||||
throw new Error('源浏览器文件被占用：请完全退出对应的浏览器（含后台进程）后重试。')
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
/** 拷贝 leveldb 目录（跳过 LOCK/LOG 运行时文件），目标已存在时先清空 */
|
||||
function copyLevelDbDir(from: string, to: string): boolean {
|
||||
if (!existsSync(from)) return false
|
||||
const entries = readdirSync(from, { withFileTypes: true })
|
||||
const files = entries.filter((e) => e.isFile() && e.name !== 'LOCK' && !e.name.startsWith('LOG'))
|
||||
if (files.length === 0) return false
|
||||
rmSync(to, { recursive: true, force: true })
|
||||
mkdirSync(to, { recursive: true })
|
||||
for (const file of files) {
|
||||
copyFileSafe(join(from, file.name), join(to, file.name))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
interface SourceKeyInfo {
|
||||
key: string | null
|
||||
/** app-bound（V20）等不可移植场景的说明 */
|
||||
note?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取源浏览器 Local State 里的 os_crypt.encrypted_key 并判断可移植性。
|
||||
* Windows：前缀 DPAPI 可移植；V20 为 app-bound 加密（Chrome/Edge 127+），
|
||||
* 其他进程无法解开，明确降级。macOS/Linux：密钥在系统钥匙串/KWallet，跳过。
|
||||
*/
|
||||
function readSourceKey(userDataDir: string): SourceKeyInfo {
|
||||
if (process.platform !== 'win32') {
|
||||
return { key: null, note: '当前平台的浏览器 Cookie 密钥绑定系统钥匙串，暂不支持迁移 Cookie（LocalStorage 登录态仍会导入）。' }
|
||||
}
|
||||
try {
|
||||
const raw = JSON.parse(readFileSync(join(userDataDir, 'Local State'), 'utf8')) as {
|
||||
os_crypt?: { encrypted_key?: string }
|
||||
}
|
||||
const encoded = raw.os_crypt?.encrypted_key
|
||||
if (!encoded) return { key: null, note: '未找到源浏览器 Cookie 密钥，跳过 Cookie 迁移。' }
|
||||
const bytes = Buffer.from(encoded, 'base64')
|
||||
if (bytes.subarray(0, 3).toString('utf8') === 'V20') {
|
||||
return { key: null, note: '该浏览器使用了应用绑定加密（app-bound encryption），无法第三方迁移 Cookie；已导入 Local Storage，首次访问各站点仍需登录一次。' }
|
||||
}
|
||||
if (bytes.subarray(0, 5).toString('utf8') !== 'DPAPI') {
|
||||
return { key: null, note: '源浏览器 Cookie 密钥格式不受支持，跳过 Cookie 迁移。' }
|
||||
}
|
||||
return { key: encoded }
|
||||
} catch {
|
||||
return { key: null, note: '读取源浏览器 Local State 失败，跳过 Cookie 迁移。' }
|
||||
}
|
||||
}
|
||||
|
||||
/** 把源密钥移植进本应用根 Local State（首次会备份原文件） */
|
||||
function transplantKey(sourceKey: string): void {
|
||||
const appLocalState = join(app.getPath('userData'), 'Local State')
|
||||
const backup = `${appLocalState}.aso-backup`
|
||||
let doc: Record<string, unknown> = {}
|
||||
if (existsSync(appLocalState)) {
|
||||
try {
|
||||
doc = JSON.parse(readFileSync(appLocalState, 'utf8')) as Record<string, unknown>
|
||||
if (!existsSync(backup)) copyFileSync(appLocalState, backup)
|
||||
} catch {
|
||||
doc = {}
|
||||
}
|
||||
}
|
||||
const osCrypt = (doc.os_crypt ?? {}) as Record<string, unknown>
|
||||
osCrypt.encrypted_key = sourceKey
|
||||
doc.os_crypt = osCrypt
|
||||
writeFileSync(appLocalState, JSON.stringify(doc, null, 2), 'utf8')
|
||||
}
|
||||
|
||||
/** 导入指定浏览器的 profile 配置到受管浏览器分区 */
|
||||
export function importBrowserProfile(sourceId: string, profileDir: string): BrowserImportResult {
|
||||
const browser = getBrowserController()
|
||||
if (browser.hasTabs) {
|
||||
throw new Error('请先关闭内置浏览器的所有标签页，再导入浏览器配置。')
|
||||
}
|
||||
const root = sourceRoot(sourceId)
|
||||
if (!root || !existsSync(root)) throw new Error('未找到该浏览器的用户数据目录。')
|
||||
// 只接受该根目录下的直接子目录名，拒绝任意路径注入
|
||||
const dirName = profileDir.split(/[\\/]/).pop() ?? ''
|
||||
const sourceProfile = join(root, dirName)
|
||||
if (dirName !== profileDir || !existsSync(join(sourceProfile, 'Preferences'))) {
|
||||
throw new Error('profile 目录不存在。')
|
||||
}
|
||||
const sourceName = SOURCE_ROOTS.find((s) => s.id === sourceId)?.name ?? sourceId
|
||||
const target = partitionDir()
|
||||
|
||||
// 1) Local Storage（明文 leveldb，直接迁移）
|
||||
let localStorage = false
|
||||
try {
|
||||
localStorage = copyLevelDbDir(join(sourceProfile, 'Local Storage', 'leveldb'), join(target, 'Local Storage', 'leveldb'))
|
||||
} catch (err) {
|
||||
throw new Error(err instanceof Error ? err.message : `Local Storage 迁移失败: ${String(err)}`)
|
||||
}
|
||||
|
||||
// 2) Cookies + 密钥移植
|
||||
let cookies = false
|
||||
let cookieNote: string | undefined
|
||||
const keyInfo = readSourceKey(root)
|
||||
const cookieDb = existsSync(join(sourceProfile, 'Network', 'Cookies'))
|
||||
? join(sourceProfile, 'Network', 'Cookies')
|
||||
: existsSync(join(sourceProfile, 'Cookies'))
|
||||
? join(sourceProfile, 'Cookies')
|
||||
: null
|
||||
if (keyInfo.key && cookieDb) {
|
||||
try {
|
||||
transplantKey(keyInfo.key)
|
||||
mkdirSync(join(target, 'Network'), { recursive: true })
|
||||
copyFileSafe(cookieDb, join(target, 'Network', 'Cookies'))
|
||||
for (const suffix of ['-wal', '-shm']) {
|
||||
if (existsSync(`${cookieDb}${suffix}`)) copyFileSafe(`${cookieDb}${suffix}`, join(target, 'Network', `Cookies${suffix}`))
|
||||
}
|
||||
cookies = true
|
||||
} catch (err) {
|
||||
cookieNote = err instanceof Error ? err.message : 'Cookie 迁移失败。'
|
||||
}
|
||||
} else {
|
||||
cookieNote = keyInfo.note ?? '源浏览器没有可迁移的 Cookie 数据库。'
|
||||
}
|
||||
|
||||
const parts: string[] = []
|
||||
if (cookies) parts.push('Cookie（站点登录态）')
|
||||
if (localStorage) parts.push('LocalStorage')
|
||||
if (!cookies && !localStorage) {
|
||||
return { ok: false, cookies, localStorage, cookieNote, message: `未在 ${sourceName}（${dirName}）中找到可迁移的登录数据。` }
|
||||
}
|
||||
|
||||
browser.setImportedInfo({
|
||||
browser: sourceName,
|
||||
profile: readProfileName(sourceProfile),
|
||||
at: new Date().toISOString(),
|
||||
cookies,
|
||||
localStorage,
|
||||
note: cookieNote
|
||||
})
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
cookies,
|
||||
localStorage,
|
||||
cookieNote,
|
||||
message: `已从 ${sourceName} 导入 ${parts.join(' + ')}。${cookieNote ?? ''}${
|
||||
cookies ? '若登录态未生效，重启应用后重试。' : ''
|
||||
}`
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { app, BrowserWindow, ipcMain, Menu, shell } from 'electron'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { registerIpc } from './ipc'
|
||||
import { getBrowserController } from './browser/browser-controller'
|
||||
|
||||
process.env.APP_ROOT = join(__dirname, '..')
|
||||
|
||||
@@ -71,9 +72,13 @@ async function createWindow(): Promise<void> {
|
||||
})
|
||||
})
|
||||
|
||||
// 窗口关闭时未决审批一律拒绝，避免 agent 永久挂起
|
||||
// 受管浏览器的原生 WebContentsView 挂在主窗口 contentView 上
|
||||
getBrowserController().setOwnerWindow(win)
|
||||
|
||||
// 窗口关闭时未决审批一律拒绝，避免 agent 永久挂起；受管浏览器同步清理
|
||||
win.on('closed', () => {
|
||||
agentService.approvals.settleAll(false)
|
||||
getBrowserController().handleWindowClosed()
|
||||
})
|
||||
|
||||
// 外部链接交给系统默认浏览器，不在应用内开新窗口
|
||||
|
||||
@@ -2,6 +2,8 @@ import { BrowserWindow, dialog, ipcMain, shell } from 'electron'
|
||||
import { mkdirSync } from 'node:fs'
|
||||
import { userInfo } from 'node:os'
|
||||
import { AgentService } from './pi/agent-service'
|
||||
import { getBrowserController } from './browser/browser-controller'
|
||||
import { detectBrowserSources, importBrowserProfile } from './browser/browser-profile'
|
||||
|
||||
/**
|
||||
* 注册 IPC 通道并把 agent 事件转发到所有渲染窗口。
|
||||
@@ -257,5 +259,70 @@ export function registerIpc(): AgentService {
|
||||
return service.pickAttachments(win ?? undefined)
|
||||
})
|
||||
|
||||
// ---- 内置受管浏览器 ----
|
||||
const browser = getBrowserController()
|
||||
// 面板状态广播（与 agent:event 同样转发到所有窗口）
|
||||
browser.onState((state) => {
|
||||
for (const win of BrowserWindow.getAllWindows()) {
|
||||
if (!win.isDestroyed()) win.webContents.send('browser:event', { type: 'state', state })
|
||||
}
|
||||
})
|
||||
|
||||
const browserPass = <T,>(fn: () => T): T => {
|
||||
try {
|
||||
return fn()
|
||||
} catch (err) {
|
||||
throw new Error(err instanceof Error ? err.message : String(err))
|
||||
}
|
||||
}
|
||||
|
||||
ipcMain.handle('browser:getState', () => browser.getState())
|
||||
ipcMain.handle('browser:navigate', (_e, input: unknown) =>
|
||||
browserPass(() => browser.navigateDisplay(typeof input === 'string' ? input : ''))
|
||||
)
|
||||
ipcMain.handle('browser:goBack', () => browserPass(() => browser.goBack()))
|
||||
ipcMain.handle('browser:goForward', () => browserPass(() => browser.goForward()))
|
||||
ipcMain.handle('browser:reload', () => browserPass(() => browser.reload()))
|
||||
ipcMain.handle('browser:newTab', (_e, url: unknown) =>
|
||||
browserPass(() => browser.newTab(typeof url === 'string' && url ? url : undefined, false))
|
||||
)
|
||||
ipcMain.handle('browser:selectTab', (_e, tabId: unknown) =>
|
||||
browserPass(() => browser.selectTab(typeof tabId === 'string' ? tabId : ''))
|
||||
)
|
||||
ipcMain.handle('browser:closeTab', (_e, tabId: unknown) =>
|
||||
browserPass(() => browser.closeTab(typeof tabId === 'string' ? tabId : undefined))
|
||||
)
|
||||
ipcMain.handle('browser:close', () => browserPass(() => browser.close()))
|
||||
ipcMain.handle('browser:acknowledgeRisk', () => browserPass(() => browser.acknowledgeRisk()))
|
||||
// 渲染层 BrowserSlot 量出的占位矩形（原生 WebContentsView 对齐到 DOM 布局）
|
||||
ipcMain.handle('browser:layout', (_e, layout: unknown) => {
|
||||
const l = layout as { tabId?: unknown; visible?: unknown; revision?: unknown; bounds?: unknown }
|
||||
const { tabId, visible, revision } = l
|
||||
const bounds = l.bounds as { x?: unknown; y?: unknown; width?: unknown; height?: unknown } | undefined
|
||||
const x = bounds?.x
|
||||
const y = bounds?.y
|
||||
const width = bounds?.width
|
||||
const height = bounds?.height
|
||||
if (
|
||||
typeof tabId !== 'string' ||
|
||||
typeof visible !== 'boolean' ||
|
||||
typeof revision !== 'number' ||
|
||||
typeof x !== 'number' ||
|
||||
typeof y !== 'number' ||
|
||||
typeof width !== 'number' ||
|
||||
typeof height !== 'number'
|
||||
) {
|
||||
throw new Error('invalid browser layout')
|
||||
}
|
||||
return browserPass(() => browser.applyLayout({ tabId, visible, revision, bounds: { x, y, width, height } }))
|
||||
})
|
||||
|
||||
// ---- 本地浏览器配置导入 ----
|
||||
ipcMain.handle('browser:detectSources', () => browserPass(() => detectBrowserSources()))
|
||||
ipcMain.handle('browser:importProfile', (_e, sourceId: unknown, profileDir: unknown) => {
|
||||
if (typeof sourceId !== 'string' || typeof profileDir !== 'string') throw new Error('invalid import source')
|
||||
return browserPass(() => importBrowserProfile(sourceId, profileDir))
|
||||
})
|
||||
|
||||
return service
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
} from '@earendil-works/pi-coding-agent'
|
||||
import { ApprovalService, createApprovalExtension } from './approval'
|
||||
import { demoTools, TOOLS_REQUIRING_APPROVAL } from '../tools/demo-tools'
|
||||
import { browserTools } from '../tools/browser-tools'
|
||||
import type {
|
||||
AgentUiEvent,
|
||||
AppSettings,
|
||||
@@ -269,7 +270,7 @@ export class AgentService {
|
||||
services,
|
||||
sessionManager,
|
||||
sessionStartEvent,
|
||||
customTools: demoTools
|
||||
customTools: [...demoTools, ...browserTools]
|
||||
})
|
||||
return { ...result, services, diagnostics: services.diagnostics }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
import { defineTool } from '@earendil-works/pi-coding-agent'
|
||||
import { Type } from '@earendil-works/pi-ai'
|
||||
import { getBrowserController } from '../browser/browser-controller'
|
||||
|
||||
/**
|
||||
* 内置受管浏览器工具（进程内一等工具，非 MCP）。
|
||||
*
|
||||
* 使用范式（与 Playwright MCP / Proma 一致的 AX 快照 + ref 体系）：
|
||||
* 1. browser_observe / browser_find 拿到带 ref 的可交互元素快照；
|
||||
* 2. 用 ref 调 click / fill / act 等操作；页面变化后旧 ref 失效，必须重新观察；
|
||||
* 3. 语义结构足够时优先 observe，需要视觉验证时才 browser_screenshot。
|
||||
*/
|
||||
|
||||
const TAB_ID_PARAM = Type.Optional(
|
||||
Type.String({ description: '可选，目标标签页 ID。默认为 Agent 工作标签页（与用户可见面板同步）。' })
|
||||
)
|
||||
|
||||
export const browserTools = [
|
||||
defineTool({
|
||||
name: 'browser_navigate',
|
||||
label: '打开网页',
|
||||
description:
|
||||
'在内置受管浏览器中打开 URL 或搜索词。完整 URL、裸域名、localhost 与 IP 直达；普通文本用 Google 搜索。支持 about:blank 空白页。下载与弹窗会留在受管浏览器内。',
|
||||
promptSnippet: 'browser_navigate: 打开网页/搜索（URL 或关键词）',
|
||||
parameters: Type.Object({
|
||||
url: Type.String({ description: 'URL、域名或搜索关键词' }),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().navigate(params.url, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify(state.tabs.find((t) => t.tabId === state.activeTabId) ?? state) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_observe',
|
||||
label: '观察网页',
|
||||
description:
|
||||
'读取当前页面的 URL、标题与紧凑可访问性快照（元素带 ref，标注 editable）。默认返回 240 个元素（约 2/3 可交互优先）。每次 observe/find 或页面导航后旧 ref 全部失效，操作必须使用最新一次观察结果中的 ref。',
|
||||
promptSnippet: 'browser_observe: 读页面结构拿 ref（先观察再操作）',
|
||||
parameters: Type.Object({
|
||||
maxElements: Type.Optional(
|
||||
Type.Number({ minimum: 20, maximum: 400, description: '返回元素上限，默认 240；仅当目标不在快照中时才提高到 400' })
|
||||
),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const observation = await getBrowserController().observe(params.tabId, params.maxElements)
|
||||
return { content: [{ type: 'text', text: JSON.stringify(observation) }], details: observation }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_find',
|
||||
label: '语义定位元素',
|
||||
description:
|
||||
'按可访问性 role 和/或 name 定位少量新 ref，不返回整页快照。适合 observe 结果过大或目标被截掉时使用。与 observe 一样会作废同标签页的全部旧 ref。',
|
||||
promptSnippet: 'browser_find: 按 role/name 精准找元素',
|
||||
parameters: Type.Object({
|
||||
role: Type.Optional(Type.String({ description: '可访问性 role，如 button/textbox/link/checkbox' })),
|
||||
name: Type.Optional(Type.String({ description: '可访问性名称查询' })),
|
||||
exact: Type.Optional(Type.Boolean({ description: '精确匹配 name（默认包含匹配）' })),
|
||||
maxResults: Type.Optional(Type.Number({ minimum: 1, maximum: 50, description: '最多返回条数，默认 20' })),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const result = await getBrowserController().find(
|
||||
{ role: params.role, name: params.name, exact: params.exact, maxResults: params.maxResults },
|
||||
params.tabId
|
||||
)
|
||||
return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_click',
|
||||
label: '点击元素',
|
||||
description: '点击最新 observe/find 结果中的指定 ref。页面会短暂高亮目标。点击导致的重渲染会让旧 ref 失效，操作后需重新观察。',
|
||||
promptSnippet: 'browser_click: 点击 ref',
|
||||
parameters: Type.Object({ ref: Type.String({ description: '元素引用（来自 observe/find）' }), tabId: TAB_ID_PARAM }),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().click(params.ref, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ clicked: params.ref, url: state.tabs.find((t) => t.tabId === state.activeTabId)?.url }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_act',
|
||||
label: '点击并等待',
|
||||
description:
|
||||
'点击 ref 后在同一次串行操作中等待一个固定条件（URL 片段/可见文本/CSS selector）。已知点击后的预期状态时优先用它，避免 click+wait 两次调用。',
|
||||
promptSnippet: 'browser_act: 点击 + 等待条件（原子操作）',
|
||||
parameters: Type.Object({
|
||||
ref: Type.String({ description: '元素引用（来自 observe/find）' }),
|
||||
waitFor: Type.Optional(
|
||||
Type.Object({
|
||||
kind: Type.Union([Type.Literal('url'), Type.Literal('text'), Type.Literal('selector')]),
|
||||
value: Type.String({ description: 'URL 片段 / 可见文本 / CSS selector' })
|
||||
})
|
||||
),
|
||||
timeoutMs: Type.Optional(Type.Number({ minimum: 250, maximum: 30000, description: '等待超时毫秒，默认 10000' })),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const result = await getBrowserController().act(params.ref, params.waitFor, params.timeoutMs, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ clicked: params.ref, matched: result.matched }) }], details: result.state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_fill',
|
||||
label: '填写输入框',
|
||||
description:
|
||||
'整体替换指定 ref 的 input/textarea/contenteditable 内容（先全选再插入，支持空格、标点、Unicode 与换行）。完整消息、搜索词、多行文本都优先用它。',
|
||||
promptSnippet: 'browser_fill: 替换输入框内容',
|
||||
parameters: Type.Object({
|
||||
ref: Type.String({ description: '可编辑字段的 ref（observe 结果中 editable=true）' }),
|
||||
text: Type.String({ description: '要输入的完整文本，≤10000 字符' }),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().fill(params.ref, params.text, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ filled: params.ref, chars: Array.from(params.text).length }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_press',
|
||||
label: '按键',
|
||||
description:
|
||||
'按导航键（Enter/Tab/Escape/方向键/PageDown 等）或向已聚焦的编辑器一次性插入完整文本。字段有 ref 时优先 browser_fill。',
|
||||
promptSnippet: 'browser_press: Enter/Tab 等按键或向焦点插入文本',
|
||||
parameters: Type.Object({ key: Type.String({ description: '导航键名，或要插入的完整文本' }), tabId: TAB_ID_PARAM }),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().press(params.key, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ pressed: params.key }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_hover',
|
||||
label: '悬停元素',
|
||||
description: '把指针移到指定 ref 上（真实 mouseMoved 事件），用于展开悬浮菜单/提示，随后需重新观察再点击新内容。',
|
||||
promptSnippet: 'browser_hover: 悬停展开菜单',
|
||||
parameters: Type.Object({ ref: Type.String({ description: '元素引用' }), tabId: TAB_ID_PARAM }),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().hover(params.ref, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ hovered: params.ref }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_scroll',
|
||||
label: '滚动页面',
|
||||
description:
|
||||
'滚动页面或 CSS selector 指定的内部滚动容器（SPA 信息流常在内部容器里，导航键滚不动）。返回前后 scrollTop 指标，用 moved 核验是否真的滚动。',
|
||||
promptSnippet: 'browser_scroll: 滚动页面/内部容器',
|
||||
parameters: Type.Object({
|
||||
deltaY: Type.Optional(Type.Number({ description: '滚动像素，正向下负向上，默认 600' })),
|
||||
position: Type.Optional(Type.Union([Type.Literal('top'), Type.Literal('bottom')], { description: '直接滚到顶部/底部（优先于 deltaY）' })),
|
||||
selector: Type.Optional(Type.String({ description: '内部滚动容器的 CSS selector，缺省滚主文档' })),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const result = await getBrowserController().scroll(
|
||||
{ deltaY: params.deltaY, position: params.position, selector: params.selector },
|
||||
params.tabId
|
||||
)
|
||||
return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_extract',
|
||||
label: '提取页面文本',
|
||||
description:
|
||||
'从页面正文或 selector 区域提取受限长度的纯文本，替代为读内容而执行 JS。优先传 selector 限定正文/列表区域，避免导航栏与页脚噪声。',
|
||||
promptSnippet: 'browser_extract: 提取正文/区域文本',
|
||||
parameters: Type.Object({
|
||||
selector: Type.Optional(Type.String({ description: '限定提取区域的 CSS selector，缺省取 body' })),
|
||||
maxChars: Type.Optional(Type.Number({ description: '最多提取字符数，默认 50000' })),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const result = await getBrowserController().extract({ selector: params.selector, maxChars: params.maxChars }, params.tabId)
|
||||
return { content: [{ type: 'text', text: result.text }], details: { url: result.url, title: result.title, truncated: result.truncated } }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_wait_for',
|
||||
label: '等待页面状态',
|
||||
description: '等待一个固定条件：URL 片段、可见文本或 CSS selector。超时返回 matched=false，不执行任意 JavaScript。',
|
||||
promptSnippet: 'browser_wait_for: 等待 url/text/selector 条件',
|
||||
parameters: Type.Object({
|
||||
kind: Type.Union([Type.Literal('url'), Type.Literal('text'), Type.Literal('selector')]),
|
||||
value: Type.String({ minLength: 1, maxLength: 2000, description: 'URL 片段 / 可见文本 / CSS selector' }),
|
||||
timeoutMs: Type.Optional(Type.Number({ minimum: 250, maximum: 30000, description: '超时毫秒，默认 10000' })),
|
||||
tabId: TAB_ID_PARAM
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const result = await getBrowserController().waitFor({ kind: params.kind, value: params.value }, params.timeoutMs, params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_screenshot',
|
||||
label: '截图',
|
||||
description: '截取当前页面。语义结构（observe）足够时不要截图；需要视觉验证、布局或渲染证据时使用。',
|
||||
promptSnippet: 'browser_screenshot: 视觉验证用',
|
||||
parameters: Type.Object({ tabId: TAB_ID_PARAM }),
|
||||
async execute(_toolCallId, params) {
|
||||
const shot = await getBrowserController().screenshot(params.tabId)
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'image' as const,
|
||||
data: shot.dataUrl.replace(/^data:image\/\w+;base64,/, ''),
|
||||
mimeType: 'image/png'
|
||||
},
|
||||
{ type: 'text', text: JSON.stringify({ url: shot.url, title: shot.title }) }
|
||||
],
|
||||
details: { url: shot.url, title: shot.title }
|
||||
}
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_list_tabs',
|
||||
label: '列出标签页',
|
||||
description: '列出受管浏览器的全部标签页（tabId/URL/标题/加载状态）与当前激活页。操作其他标签页时先调它拿 tabId。',
|
||||
promptSnippet: 'browser_list_tabs: 查看标签页列表',
|
||||
parameters: Type.Object({}),
|
||||
async execute() {
|
||||
const state = getBrowserController().listTabs()
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ tabs: state.tabs, activeTabId: state.activeTabId }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_new_tab',
|
||||
label: '新建标签页',
|
||||
description: '新建 Agent 工作标签页并激活到用户可见的浏览器面板；完成后不再需要的标签页应主动 browser_close_tab。',
|
||||
promptSnippet: 'browser_new_tab: 新建工作标签页',
|
||||
parameters: Type.Object({
|
||||
url: Type.Optional(Type.String({ description: '初始地址（URL/域名/搜索词），缺省开空白起始页' }))
|
||||
}),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().newTab(params.url, true)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ tabId: state.activeTabId, tabs: state.tabs.length }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_select_tab',
|
||||
label: '切换标签页',
|
||||
description: '把指定标签页设为 Agent 工作页，并同步激活到用户可见面板。后续不带 tabId 的操作都作用于它。',
|
||||
promptSnippet: 'browser_select_tab: 切换工作标签页',
|
||||
parameters: Type.Object({ tabId: Type.String({ description: 'browser_list_tabs 返回的标签页 ID' }) }),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().selectTab(params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ activeTabId: state.activeTabId }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_close_tab',
|
||||
label: '关闭标签页',
|
||||
description: '关闭指定标签页（缺省关闭当前工作页）。绝不关闭用户手动打开的标签页。',
|
||||
promptSnippet: 'browser_close_tab: 关闭工作标签页',
|
||||
parameters: Type.Object({ tabId: Type.Optional(Type.String({ description: '要关闭的标签页 ID，缺省当前工作页' })) }),
|
||||
async execute(_toolCallId, params) {
|
||||
const state = await getBrowserController().closeTab(params.tabId)
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ closed: params.tabId ?? 'current', remaining: state.tabs.length }) }], details: state }
|
||||
}
|
||||
}),
|
||||
|
||||
defineTool({
|
||||
name: 'browser_close',
|
||||
label: '关闭浏览器',
|
||||
description: '关闭整个内置受管浏览器会话（销毁全部标签页）。任务结束、不再需要网页时主动调用。',
|
||||
promptSnippet: 'browser_close: 关闭受管浏览器',
|
||||
parameters: Type.Object({}),
|
||||
async execute() {
|
||||
const state = getBrowserController().close()
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ closed: true }) }], details: state }
|
||||
}
|
||||
})
|
||||
]
|
||||
|
||||
export const browserToolNames = new Set(browserTools.map((tool) => tool.name))
|
||||
@@ -3,6 +3,10 @@ import type {
|
||||
AgentUiEvent,
|
||||
AppSettings,
|
||||
AttachmentInfo,
|
||||
BrowserImportResult,
|
||||
BrowserSource,
|
||||
BrowserUiEvent,
|
||||
BrowserUiState,
|
||||
ChatResources,
|
||||
CustomProviderConfig,
|
||||
ModelCatalog,
|
||||
@@ -72,6 +76,33 @@ const api = {
|
||||
return () => {
|
||||
ipcRenderer.removeListener('agent:event', listener)
|
||||
}
|
||||
},
|
||||
// ---- 内置受管浏览器 ----
|
||||
browserGetState: (): Promise<BrowserUiState> => ipcRenderer.invoke('browser:getState'),
|
||||
browserNavigate: (input: string): Promise<BrowserUiState> => ipcRenderer.invoke('browser:navigate', input),
|
||||
browserGoBack: (): Promise<BrowserUiState> => ipcRenderer.invoke('browser:goBack'),
|
||||
browserGoForward: (): Promise<BrowserUiState> => ipcRenderer.invoke('browser:goForward'),
|
||||
browserReload: (): Promise<BrowserUiState> => ipcRenderer.invoke('browser:reload'),
|
||||
browserNewTab: (url?: string): Promise<BrowserUiState> => ipcRenderer.invoke('browser:newTab', url),
|
||||
browserSelectTab: (tabId: string): Promise<BrowserUiState> => ipcRenderer.invoke('browser:selectTab', tabId),
|
||||
browserCloseTab: (tabId?: string): Promise<BrowserUiState> => ipcRenderer.invoke('browser:closeTab', tabId),
|
||||
browserClose: (): Promise<BrowserUiState> => ipcRenderer.invoke('browser:close'),
|
||||
browserAcknowledgeRisk: (): Promise<void> => ipcRenderer.invoke('browser:acknowledgeRisk'),
|
||||
browserSetLayout: (layout: {
|
||||
tabId: string
|
||||
visible: boolean
|
||||
revision: number
|
||||
bounds: { x: number; y: number; width: number; height: number }
|
||||
}): Promise<void> => ipcRenderer.invoke('browser:layout', layout),
|
||||
browserDetectSources: (): Promise<BrowserSource[]> => ipcRenderer.invoke('browser:detectSources'),
|
||||
browserImportProfile: (sourceId: string, profileDir: string): Promise<BrowserImportResult> =>
|
||||
ipcRenderer.invoke('browser:importProfile', sourceId, profileDir),
|
||||
onBrowserEvent: (callback: (event: BrowserUiEvent) => void): (() => void) => {
|
||||
const listener = (_e: Electron.IpcRendererEvent, event: BrowserUiEvent): void => callback(event)
|
||||
ipcRenderer.on('browser:event', listener)
|
||||
return () => {
|
||||
ipcRenderer.removeListener('browser:event', listener)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import type { JSX } from 'react'
|
||||
import { useAgent } from './hooks/useAgent'
|
||||
import { useBrowser } from './hooks/useBrowser'
|
||||
import { SidebarNav } from './components/SidebarNav'
|
||||
import { TitleBar } from './components/TitleBar'
|
||||
import { ChatView } from './components/ChatView'
|
||||
@@ -8,6 +9,7 @@ import { BottomSlot } from './components/BottomSlot'
|
||||
import { Composer } from './components/Composer'
|
||||
import { SettingsModal } from './components/SettingsModal'
|
||||
import { Search } from './components/Search'
|
||||
import { BrowserPanel } from './components/browser/BrowserPanel'
|
||||
import { applyThemeColor } from './lib/theme'
|
||||
|
||||
export default function App(): JSX.Element {
|
||||
@@ -31,6 +33,8 @@ export default function App(): JSX.Element {
|
||||
const prefillRef = useRef<string | null>(null)
|
||||
const [settingsOpen, setSettingsOpen] = useState(false)
|
||||
const [searchOpen, setSearchOpen] = useState(false)
|
||||
const [browserOpen, setBrowserOpen] = useState(false)
|
||||
const browser = useBrowser()
|
||||
|
||||
// 启动时应用持久化的主题色
|
||||
useEffect(() => {
|
||||
@@ -96,7 +100,13 @@ export default function App(): JSX.Element {
|
||||
onOpenWorkspace={openWorkspace}
|
||||
/>
|
||||
<main className="relative flex min-w-0 flex-1 flex-col">
|
||||
<TitleBar title={sessionTitle} />
|
||||
<TitleBar
|
||||
title={sessionTitle}
|
||||
browserOpen={browserOpen}
|
||||
onToggleBrowser={() => setBrowserOpen((open) => !open)}
|
||||
/>
|
||||
<div className="flex min-h-0 flex-1">
|
||||
<div className="relative flex min-w-0 flex-1 flex-col">
|
||||
<ChatView
|
||||
messages={state.messages}
|
||||
tools={state.tools}
|
||||
@@ -131,6 +141,24 @@ export default function App(): JSX.Element {
|
||||
/>
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
{browserOpen && (
|
||||
<BrowserPanel
|
||||
state={browser.state}
|
||||
suspended={settingsOpen || searchOpen}
|
||||
onNavigate={browser.navigate}
|
||||
onGoBack={browser.goBack}
|
||||
onGoForward={browser.goForward}
|
||||
onReload={browser.reload}
|
||||
onNewTab={browser.newTab}
|
||||
onSelectTab={browser.selectTab}
|
||||
onCloseTab={browser.closeTab}
|
||||
onAcknowledgeRisk={browser.acknowledgeRisk}
|
||||
onDetectSources={browser.detectSources}
|
||||
onImportProfile={browser.importProfile}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</main>
|
||||
<Search
|
||||
open={searchOpen}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useEffect, useState, type JSX } from 'react'
|
||||
import { LuGlobe } from 'react-icons/lu'
|
||||
|
||||
/**
|
||||
* 自绘标题栏（配合主进程 titleBarStyle:'hidden'）。
|
||||
@@ -6,7 +7,15 @@ import { useEffect, useState, type JSX } from 'react'
|
||||
* 右侧自绘窗口控制按钮（最小化/最大化还原/关闭），高度 40px。
|
||||
* 不使用系统 WCO overlay —— 其原生背景会盖住标题栏分隔线。
|
||||
*/
|
||||
export function TitleBar({ title }: { title?: string }): JSX.Element {
|
||||
export function TitleBar({
|
||||
title,
|
||||
browserOpen,
|
||||
onToggleBrowser
|
||||
}: {
|
||||
title?: string
|
||||
browserOpen: boolean
|
||||
onToggleBrowser: () => void
|
||||
}): JSX.Element {
|
||||
const name = title?.trim()
|
||||
const [maximized, setMaximized] = useState(false)
|
||||
|
||||
@@ -15,7 +24,20 @@ export function TitleBar({ title }: { title?: string }): JSX.Element {
|
||||
return (
|
||||
<div className="relative z-20 flex h-10 shrink-0 select-none items-center border-b border-zinc-200 bg-white pl-4 [-webkit-app-region:drag]">
|
||||
<div className="min-w-0 truncate text-[12.5px] font-medium text-zinc-800">{name || '新会话'}</div>
|
||||
<div className="ml-auto flex h-full shrink-0 [-webkit-app-region:no-drag]">
|
||||
<div className="ml-auto flex h-full shrink-0 items-center gap-1 pr-1.5 [-webkit-app-region:no-drag]">
|
||||
<button
|
||||
type="button"
|
||||
aria-label={browserOpen ? '关闭内置浏览器' : '打开内置浏览器'}
|
||||
title="内置浏览器"
|
||||
onClick={onToggleBrowser}
|
||||
className={`flex h-7 w-[30px] items-center justify-center rounded transition-colors ${
|
||||
browserOpen ? 'bg-accent-tint text-accent-ink' : 'text-zinc-500 hover:bg-zinc-100 hover:text-zinc-700'
|
||||
}`}
|
||||
>
|
||||
<LuGlobe size={14} />
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex h-full shrink-0 [-webkit-app-region:no-drag]">
|
||||
<button
|
||||
type="button"
|
||||
aria-label="最小化"
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
import { useEffect, useMemo, useRef, useState, type JSX } from 'react'
|
||||
import {
|
||||
LuArrowLeft,
|
||||
LuArrowRight,
|
||||
LuChevronDown,
|
||||
LuCircleAlert,
|
||||
LuDownload,
|
||||
LuGlobe,
|
||||
LuPlus,
|
||||
LuRotateCw,
|
||||
LuX
|
||||
} from 'react-icons/lu'
|
||||
import type { BrowserImportResult, BrowserSource, BrowserUiState } from '@shared/types'
|
||||
import { BrowserSlot } from './BrowserSlot'
|
||||
|
||||
/**
|
||||
* 内置受管浏览器右侧面板。
|
||||
*
|
||||
* 页面本体是主进程的原生 WebContentsView（盖在 DOM 之上），面板 DOM 只负责
|
||||
* 标签栏/地址栏/工具条的「chrome」部分；BrowserSlot 量出页面区域矩形后由主进程
|
||||
* 对齐原生视图。任何全屏浮层（设置/搜索/导入）出现时必须卸载 Slot 隐藏原生视图。
|
||||
*/
|
||||
|
||||
const PANEL_DEFAULT_WIDTH = 480
|
||||
const PANEL_MIN_WIDTH = 380
|
||||
const PANEL_MAX_WIDTH = 760
|
||||
|
||||
interface BrowserPanelProps {
|
||||
state: BrowserUiState | null
|
||||
/** 全屏浮层（设置/搜索）打开时暂停原生视图，避免其盖住浮层 */
|
||||
suspended: boolean
|
||||
onNavigate: (input: string) => void
|
||||
onGoBack: () => void
|
||||
onGoForward: () => void
|
||||
onReload: () => void
|
||||
onNewTab: () => void
|
||||
onSelectTab: (tabId: string) => void
|
||||
onCloseTab: (tabId: string) => void
|
||||
onAcknowledgeRisk: () => void
|
||||
onDetectSources: () => Promise<BrowserSource[]>
|
||||
onImportProfile: (sourceId: string, profileDir: string) => Promise<BrowserImportResult>
|
||||
}
|
||||
|
||||
export function BrowserPanel(props: BrowserPanelProps): JSX.Element {
|
||||
const { state, suspended } = props
|
||||
const [width, setWidth] = useState(PANEL_DEFAULT_WIDTH)
|
||||
const [address, setAddress] = useState('')
|
||||
const [importOpen, setImportOpen] = useState(false)
|
||||
const dragState = useRef<{ startX: number; startWidth: number } | null>(null)
|
||||
|
||||
const activeTab = useMemo(
|
||||
() => state?.tabs.find((t) => t.tabId === state.activeTabId) ?? null,
|
||||
[state]
|
||||
)
|
||||
|
||||
// 地址栏跟随激活页 URL（用户正在输入时不打扰）
|
||||
const addressRef = useRef<HTMLInputElement>(null)
|
||||
useEffect(() => {
|
||||
if (document.activeElement !== addressRef.current) {
|
||||
setAddress(activeTab?.url ?? '')
|
||||
}
|
||||
}, [activeTab?.url])
|
||||
|
||||
// 面板宽度拖拽（左边缘把手）
|
||||
useEffect(() => {
|
||||
const onMove = (e: PointerEvent): void => {
|
||||
if (!dragState.current) return
|
||||
const delta = dragState.current.startX - e.clientX
|
||||
setWidth(Math.min(PANEL_MAX_WIDTH, Math.max(PANEL_MIN_WIDTH, dragState.current.startWidth + delta)))
|
||||
}
|
||||
const onUp = (): void => {
|
||||
dragState.current = null
|
||||
}
|
||||
window.addEventListener('pointermove', onMove)
|
||||
window.addEventListener('pointerup', onUp)
|
||||
return () => {
|
||||
window.removeEventListener('pointermove', onMove)
|
||||
window.removeEventListener('pointerup', onUp)
|
||||
}
|
||||
}, [])
|
||||
|
||||
const startResize = (e: React.PointerEvent): void => {
|
||||
dragState.current = { startX: e.clientX, startWidth: width }
|
||||
}
|
||||
|
||||
const submitAddress = (): void => {
|
||||
const value = address.trim()
|
||||
if (value) props.onNavigate(value)
|
||||
addressRef.current?.blur()
|
||||
}
|
||||
|
||||
const riskBanner = state && !state.riskAcknowledged
|
||||
const showSlot = Boolean(activeTab) && !suspended && !importOpen
|
||||
|
||||
return (
|
||||
<aside
|
||||
className="relative flex h-full shrink-0 flex-col border-l border-zinc-200 bg-zinc-50"
|
||||
style={{ width }}
|
||||
aria-label="内置浏览器"
|
||||
>
|
||||
{/* 左边缘拖拽把手 */}
|
||||
<div
|
||||
className="absolute inset-y-0 left-0 z-10 w-1 cursor-col-resize hover:bg-accent/40"
|
||||
onPointerDown={startResize}
|
||||
role="separator"
|
||||
aria-orientation="vertical"
|
||||
/>
|
||||
|
||||
{/* 标签栏 */}
|
||||
<div className="flex h-9 shrink-0 items-center gap-1 overflow-x-auto border-b border-zinc-200 bg-white px-1.5">
|
||||
{(state?.tabs ?? []).map((tab) => (
|
||||
<button
|
||||
key={tab.tabId}
|
||||
type="button"
|
||||
onClick={() => props.onSelectTab(tab.tabId)}
|
||||
className={`group flex h-7 max-w-44 min-w-20 shrink-0 items-center gap-1 rounded px-2 text-left text-[11px] transition-colors ${
|
||||
tab.tabId === state?.activeTabId
|
||||
? 'bg-accent-tint text-accent-ink'
|
||||
: 'text-zinc-500 hover:bg-zinc-100 hover:text-zinc-700'
|
||||
}`}
|
||||
>
|
||||
{tab.loading && <LuRotateCw size={10} className="shrink-0 animate-spin" />}
|
||||
<span className="min-w-0 flex-1 truncate">{tab.title || tab.url || '新标签页'}</span>
|
||||
<span
|
||||
role="button"
|
||||
tabIndex={-1}
|
||||
aria-label="关闭标签页"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
props.onCloseTab(tab.tabId)
|
||||
}}
|
||||
className="hidden shrink-0 rounded p-0.5 hover:bg-zinc-200 group-hover:block"
|
||||
>
|
||||
<LuX size={10} />
|
||||
</span>
|
||||
</button>
|
||||
))}
|
||||
<button
|
||||
type="button"
|
||||
aria-label="新建标签页"
|
||||
onClick={props.onNewTab}
|
||||
disabled={Boolean(riskBanner)}
|
||||
className="flex size-7 shrink-0 items-center justify-center rounded text-zinc-400 transition-colors hover:bg-zinc-100 hover:text-zinc-600 disabled:opacity-40"
|
||||
>
|
||||
<LuPlus size={13} />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* 工具条 */}
|
||||
<div className="flex h-10 shrink-0 items-center gap-1 border-b border-zinc-200 bg-white px-1.5">
|
||||
<button
|
||||
type="button"
|
||||
aria-label="后退"
|
||||
onClick={props.onGoBack}
|
||||
className="flex size-7 items-center justify-center rounded text-zinc-500 transition-colors hover:bg-zinc-100 hover:text-zinc-700"
|
||||
>
|
||||
<LuArrowLeft size={14} />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="前进"
|
||||
onClick={props.onGoForward}
|
||||
className="flex size-7 items-center justify-center rounded text-zinc-500 transition-colors hover:bg-zinc-100 hover:text-zinc-700"
|
||||
>
|
||||
<LuArrowRight size={14} />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="刷新"
|
||||
onClick={props.onReload}
|
||||
className="flex size-7 items-center justify-center rounded text-zinc-500 transition-colors hover:bg-zinc-100 hover:text-zinc-700"
|
||||
>
|
||||
<LuRotateCw size={13} />
|
||||
</button>
|
||||
<div className="mx-1 flex h-7 min-w-0 flex-1 items-center gap-1.5 rounded border border-zinc-200 bg-zinc-50 px-2 focus-within:border-accent focus-within:bg-white">
|
||||
<LuGlobe size={11} className="shrink-0 text-zinc-400" />
|
||||
<input
|
||||
ref={addressRef}
|
||||
value={address}
|
||||
onChange={(e) => setAddress(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') submitAddress()
|
||||
if (e.key === 'Escape') addressRef.current?.blur()
|
||||
}}
|
||||
placeholder="输入网址或搜索…"
|
||||
spellCheck={false}
|
||||
className="min-w-0 flex-1 bg-transparent text-[12px] text-zinc-700 outline-none placeholder:text-zinc-400"
|
||||
/>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setImportOpen(true)}
|
||||
title="导入本机浏览器登录态"
|
||||
className="flex size-7 items-center justify-center rounded text-zinc-500 transition-colors hover:bg-zinc-100 hover:text-zinc-700"
|
||||
>
|
||||
<LuDownload size={13} />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* 风险告知（首次使用门禁，Agent 与用户操作都会被拦截直到确认） */}
|
||||
{riskBanner && (
|
||||
<div className="flex shrink-0 items-start gap-2 border-b border-amber-200 bg-amber-50 px-3 py-2.5 text-[11.5px] leading-relaxed text-amber-800">
|
||||
<LuCircleAlert size={14} className="mt-0.5 shrink-0" />
|
||||
<div className="min-w-0 flex-1">
|
||||
<p>
|
||||
Agent 将以你的身份操作网页。平台可能将自动化行为识别为异常，导致验证码、限流或账号风控；请勿在受管浏览器中登录支付等高敏感账号。
|
||||
</p>
|
||||
<button
|
||||
type="button"
|
||||
onClick={props.onAcknowledgeRisk}
|
||||
className="mt-1.5 rounded bg-amber-600 px-2.5 py-1 text-[11px] font-medium text-white transition-colors hover:bg-amber-700"
|
||||
>
|
||||
我已了解，开始使用
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* 页面区域：原生 WebContentsView 对齐到这里的矩形 */}
|
||||
<div className="relative min-h-0 flex-1 bg-white">
|
||||
{showSlot && activeTab ? (
|
||||
<BrowserSlot key={activeTab.tabId} tabId={activeTab.tabId} />
|
||||
) : (
|
||||
<div className="flex h-full flex-col items-center justify-center gap-3 px-8 text-center">
|
||||
<LuGlobe size={28} className="text-zinc-300" />
|
||||
{riskBanner ? (
|
||||
<p className="text-[12px] text-zinc-400">确认上方风险告知后即可打开网页</p>
|
||||
) : (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
onClick={props.onNewTab}
|
||||
className="rounded bg-accent px-3.5 py-1.5 text-[12px] font-medium text-white transition-opacity hover:opacity-90"
|
||||
>
|
||||
打开浏览器
|
||||
</button>
|
||||
<p className="text-[11.5px] leading-relaxed text-zinc-400">
|
||||
也可以直接让 Agent 帮你操作网页（browser_navigate / browser_observe 等工具）
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Agent 操作轨迹 */}
|
||||
{state?.trace && !importOpen && (
|
||||
<div className="shrink-0 truncate border-t border-zinc-200 bg-white px-3 py-1.5 text-[10.5px] text-zinc-400">
|
||||
{state.trace}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* 配置导入浮层（覆盖面板，出现时原生视图已隐藏） */}
|
||||
{importOpen && (
|
||||
<ImportOverlay
|
||||
state={state}
|
||||
onClose={() => setImportOpen(false)}
|
||||
onDetectSources={props.onDetectSources}
|
||||
onImportProfile={props.onImportProfile}
|
||||
/>
|
||||
)}
|
||||
</aside>
|
||||
)
|
||||
}
|
||||
|
||||
// ---- 配置导入浮层 ----
|
||||
|
||||
function ImportOverlay({
|
||||
state,
|
||||
onClose,
|
||||
onDetectSources,
|
||||
onImportProfile
|
||||
}: {
|
||||
state: BrowserUiState | null
|
||||
onClose: () => void
|
||||
onDetectSources: () => Promise<BrowserSource[]>
|
||||
onImportProfile: (sourceId: string, profileDir: string) => Promise<BrowserImportResult>
|
||||
}): JSX.Element {
|
||||
const [sources, setSources] = useState<BrowserSource[] | null>(null)
|
||||
const [selected, setSelected] = useState<{ sourceId: string; profileDir: string } | null>(null)
|
||||
const [busy, setBusy] = useState(false)
|
||||
const [result, setResult] = useState<BrowserImportResult | null>(null)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
onDetectSources()
|
||||
.then((list) => {
|
||||
setSources(list)
|
||||
const first = list[0]
|
||||
if (first?.profiles[0]) setSelected({ sourceId: first.id, profileDir: first.profiles[0].dir })
|
||||
})
|
||||
.catch((err) => setError(err instanceof Error ? err.message : String(err)))
|
||||
}, [onDetectSources])
|
||||
|
||||
const runImport = (): void => {
|
||||
if (!selected) return
|
||||
setBusy(true)
|
||||
setError(null)
|
||||
setResult(null)
|
||||
onImportProfile(selected.sourceId, selected.profileDir)
|
||||
.then(setResult)
|
||||
.catch((err) => setError(err instanceof Error ? err.message : String(err)))
|
||||
.finally(() => setBusy(false))
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="absolute inset-0 z-20 flex flex-col bg-white/95 backdrop-blur-[2px]">
|
||||
<div className="flex h-10 shrink-0 items-center justify-between border-b border-zinc-200 px-3">
|
||||
<div className="text-[12.5px] font-medium text-zinc-700">导入本机浏览器登录态</div>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="关闭"
|
||||
onClick={onClose}
|
||||
className="flex size-6 items-center justify-center rounded text-zinc-400 hover:bg-zinc-100 hover:text-zinc-600"
|
||||
>
|
||||
<LuX size={13} />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="min-h-0 flex-1 overflow-y-auto p-3 text-[12px] text-zinc-600">
|
||||
<p className="mb-3 leading-relaxed text-zinc-500">
|
||||
选择本机浏览器的一个配置（Profile），把其中各网站的登录态（Cookie / Local Storage）导入到内置受管浏览器。
|
||||
导入前请<span className="font-medium text-zinc-700">完全退出</span>该浏览器，并关闭本面板的所有标签页。
|
||||
</p>
|
||||
|
||||
{state?.imported && (
|
||||
<div className="mb-3 rounded border border-zinc-200 bg-zinc-50 px-2.5 py-2 text-[11.5px] text-zinc-500">
|
||||
当前导入来源：{state.imported.browser} · {state.imported.profile}
|
||||
{state.imported.cookies ? '（含 Cookie）' : ''}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{sources === null && !error && <p className="py-6 text-center text-zinc-400">正在检测本机浏览器…</p>}
|
||||
{sources?.length === 0 && <p className="py-6 text-center text-zinc-400">未检测到可导入的本机浏览器（支持 Chrome / Edge / Brave / Chromium）</p>}
|
||||
|
||||
<div className="flex flex-col gap-2">
|
||||
{sources?.map((source) => (
|
||||
<div key={source.id} className="rounded border border-zinc-200">
|
||||
<div className="flex items-center gap-1.5 px-2.5 py-2 text-[12px] font-medium text-zinc-700">
|
||||
<LuChevronDown size={12} className="text-zinc-400" />
|
||||
{source.name}
|
||||
<span className="text-[10.5px] font-normal text-zinc-400">（{source.profiles.length} 个配置）</span>
|
||||
</div>
|
||||
<div className="flex flex-col gap-1 px-2 pb-2">
|
||||
{source.profiles.map((profile) => {
|
||||
const active = selected?.sourceId === source.id && selected.profileDir === profile.dir
|
||||
return (
|
||||
<button
|
||||
key={profile.dir}
|
||||
type="button"
|
||||
onClick={() => setSelected({ sourceId: source.id, profileDir: profile.dir })}
|
||||
className={`flex items-center justify-between rounded px-2.5 py-1.5 text-left text-[11.5px] transition-colors ${
|
||||
active ? 'bg-accent-tint text-accent-ink' : 'text-zinc-600 hover:bg-zinc-100'
|
||||
}`}
|
||||
>
|
||||
<span className="min-w-0 truncate">
|
||||
{profile.name}
|
||||
<span className="ml-1.5 text-[10.5px] text-zinc-400">{profile.dir}</span>
|
||||
</span>
|
||||
{active && <span className="shrink-0 text-[10px]">已选择</span>}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="mt-3 rounded border border-red-200 bg-red-50 px-2.5 py-2 text-[11.5px] text-red-600">{error}</div>
|
||||
)}
|
||||
{result && (
|
||||
<div
|
||||
className={`mt-3 rounded border px-2.5 py-2 text-[11.5px] leading-relaxed ${
|
||||
result.ok ? 'border-green-200 bg-green-50 text-green-700' : 'border-red-200 bg-red-50 text-red-600'
|
||||
}`}
|
||||
>
|
||||
{result.message}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex shrink-0 items-center justify-end gap-2 border-t border-zinc-200 px-3 py-2.5">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
className="rounded px-3 py-1.5 text-[12px] text-zinc-500 transition-colors hover:bg-zinc-100 hover:text-zinc-700"
|
||||
>
|
||||
关闭
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
disabled={!selected || busy}
|
||||
onClick={runImport}
|
||||
className="rounded bg-accent px-3.5 py-1.5 text-[12px] font-medium text-white transition-opacity hover:opacity-90 disabled:opacity-40"
|
||||
>
|
||||
{busy ? '导入中…' : '导入登录态'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import * as React from 'react'
|
||||
|
||||
/**
|
||||
* 受管浏览器页面的 DOM 占位符。
|
||||
*
|
||||
* 原生 WebContentsView 由主进程挂载在窗口 contentView 上，天然盖在所有 DOM 之上，
|
||||
* CSS z-index 无法反转。这里量出占位矩形（ResizeObserver + 窗口 resize），经 IPC
|
||||
* 同步给主进程做 setBounds，让原生视图精确对齐到面板布局。
|
||||
*
|
||||
* revision 全局单调递增：快速切换 Tab 时旧 Slot 晚到的 IPC 不会覆盖新挂载的视图。
|
||||
*/
|
||||
|
||||
let revisionCounter = 0
|
||||
function nextRevision(): number {
|
||||
revisionCounter += 1
|
||||
return revisionCounter
|
||||
}
|
||||
|
||||
export function BrowserSlot({ tabId }: { tabId: string }): React.ReactElement {
|
||||
const ref = React.useRef<HTMLDivElement>(null)
|
||||
|
||||
React.useLayoutEffect(() => {
|
||||
const element = ref.current
|
||||
if (!element) return
|
||||
let frame = 0
|
||||
|
||||
const commit = (visible: boolean): void => {
|
||||
const rect = element.getBoundingClientRect()
|
||||
void window.api.browserSetLayout({
|
||||
tabId,
|
||||
visible: visible && rect.width > 4 && rect.height > 4,
|
||||
revision: nextRevision(),
|
||||
bounds: {
|
||||
x: Math.round(rect.x),
|
||||
y: Math.round(rect.y),
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height)
|
||||
}
|
||||
})
|
||||
}
|
||||
const publish = (immediate = false): void => {
|
||||
if (frame) cancelAnimationFrame(frame)
|
||||
if (immediate) {
|
||||
frame = 0
|
||||
commit(true)
|
||||
return
|
||||
}
|
||||
frame = requestAnimationFrame(() => {
|
||||
frame = 0
|
||||
commit(true)
|
||||
})
|
||||
}
|
||||
|
||||
const observer = new ResizeObserver(() => publish())
|
||||
observer.observe(element)
|
||||
window.addEventListener('resize', onWindowResize)
|
||||
// 挂载立即上报（不等一帧），避免快速切换时原生视图停留在隐藏状态
|
||||
publish(true)
|
||||
|
||||
function onWindowResize(): void {
|
||||
publish()
|
||||
}
|
||||
|
||||
return () => {
|
||||
observer.disconnect()
|
||||
window.removeEventListener('resize', onWindowResize)
|
||||
if (frame) cancelAnimationFrame(frame)
|
||||
void window.api.browserSetLayout({
|
||||
tabId,
|
||||
visible: false,
|
||||
revision: nextRevision(),
|
||||
bounds: { x: 0, y: 0, width: 0, height: 0 }
|
||||
})
|
||||
}
|
||||
}, [tabId])
|
||||
|
||||
return <div ref={ref} className="h-full w-full bg-white" aria-label="受管浏览器页面" />
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import type { BrowserImportResult, BrowserSource, BrowserUiState } from '@shared/types'
|
||||
|
||||
/**
|
||||
* 内置受管浏览器面板状态与操作。
|
||||
* 状态由主进程 BrowserController 广播（browser:event），这里只做订阅与动作封装。
|
||||
*/
|
||||
export function useBrowser(): {
|
||||
state: BrowserUiState | null
|
||||
navigate: (input: string) => void
|
||||
goBack: () => void
|
||||
goForward: () => void
|
||||
reload: () => void
|
||||
newTab: () => void
|
||||
selectTab: (tabId: string) => void
|
||||
closeTab: (tabId: string) => void
|
||||
closeBrowser: () => void
|
||||
acknowledgeRisk: () => void
|
||||
detectSources: () => Promise<BrowserSource[]>
|
||||
importProfile: (sourceId: string, profileDir: string) => Promise<BrowserImportResult>
|
||||
} {
|
||||
const [state, setState] = useState<BrowserUiState | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
void window.api.browserGetState().then(setState).catch(() => undefined)
|
||||
return window.api.onBrowserEvent((event) => {
|
||||
if (event.type === 'state') setState(event.state)
|
||||
})
|
||||
}, [])
|
||||
|
||||
const dispatchError = useCallback((err: unknown): void => {
|
||||
console.error('[browser]', err instanceof Error ? err.message : String(err))
|
||||
}, [])
|
||||
|
||||
const wrap = useCallback(
|
||||
<A,>(fn: (...args: A[]) => Promise<unknown>) =>
|
||||
(...args: A[]): void => {
|
||||
void fn(...args).catch(dispatchError)
|
||||
},
|
||||
[dispatchError]
|
||||
)
|
||||
|
||||
return {
|
||||
state,
|
||||
navigate: wrap((input: string) => window.api.browserNavigate(input)),
|
||||
goBack: wrap(() => window.api.browserGoBack()),
|
||||
goForward: wrap(() => window.api.browserGoForward()),
|
||||
reload: wrap(() => window.api.browserReload()),
|
||||
newTab: wrap(() => window.api.browserNewTab()),
|
||||
selectTab: wrap((tabId: string) => window.api.browserSelectTab(tabId)),
|
||||
closeTab: wrap((tabId: string) => window.api.browserCloseTab(tabId)),
|
||||
closeBrowser: wrap(() => window.api.browserClose()),
|
||||
acknowledgeRisk: wrap(() => window.api.browserAcknowledgeRisk()),
|
||||
detectSources: () => window.api.browserDetectSources(),
|
||||
importProfile: (sourceId: string, profileDir: string) => window.api.browserImportProfile(sourceId, profileDir)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,10 @@ import type {
|
||||
AgentUiEvent,
|
||||
AppSettings,
|
||||
AttachmentInfo,
|
||||
BrowserImportResult,
|
||||
BrowserSource,
|
||||
BrowserUiEvent,
|
||||
BrowserUiState,
|
||||
ChatResources,
|
||||
CustomProviderConfig,
|
||||
ModelCatalog,
|
||||
@@ -16,7 +20,19 @@ import type {
|
||||
WorkspaceMeta
|
||||
} from '@shared/types'
|
||||
|
||||
export type { AgentUiEvent, AppSettings, ModelCatalog, ModelInfo, SessionMeta, SessionState, SessionStatsInfo }
|
||||
export type {
|
||||
AgentUiEvent,
|
||||
AppSettings,
|
||||
ModelCatalog,
|
||||
ModelInfo,
|
||||
SessionMeta,
|
||||
SessionState,
|
||||
SessionStatsInfo,
|
||||
BrowserImportResult,
|
||||
BrowserSource,
|
||||
BrowserUiEvent,
|
||||
BrowserUiState
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
@@ -59,6 +75,25 @@ declare global {
|
||||
setThemeColor: (id: string) => Promise<void>
|
||||
resolveApproval: (id: string, approved: boolean) => Promise<void>
|
||||
onEvent: (callback: (event: AgentUiEvent) => void) => () => void
|
||||
browserGetState: () => Promise<BrowserUiState>
|
||||
browserNavigate: (input: string) => Promise<BrowserUiState>
|
||||
browserGoBack: () => Promise<BrowserUiState>
|
||||
browserGoForward: () => Promise<BrowserUiState>
|
||||
browserReload: () => Promise<BrowserUiState>
|
||||
browserNewTab: (url?: string) => Promise<BrowserUiState>
|
||||
browserSelectTab: (tabId: string) => Promise<BrowserUiState>
|
||||
browserCloseTab: (tabId?: string) => Promise<BrowserUiState>
|
||||
browserClose: () => Promise<BrowserUiState>
|
||||
browserAcknowledgeRisk: () => Promise<void>
|
||||
browserSetLayout: (layout: {
|
||||
tabId: string
|
||||
visible: boolean
|
||||
revision: number
|
||||
bounds: { x: number; y: number; width: number; height: number }
|
||||
}) => Promise<void>
|
||||
browserDetectSources: () => Promise<BrowserSource[]>
|
||||
browserImportProfile: (sourceId: string, profileDir: string) => Promise<BrowserImportResult>
|
||||
onBrowserEvent: (callback: (event: BrowserUiEvent) => void) => () => void
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,6 +182,66 @@ export interface SendImage {
|
||||
* 转发到渲染进程的 agent 事件。
|
||||
* 已归一化为可结构化克隆的普通 JSON（message 原样透传，渲染层防御性读取）。
|
||||
*/
|
||||
// ===== 内置受管浏览器 =====
|
||||
|
||||
/** 浏览器标签页（主进程 → 渲染层面板） */
|
||||
export interface BrowserTabInfo {
|
||||
tabId: string
|
||||
url: string
|
||||
title: string
|
||||
loading: boolean
|
||||
}
|
||||
|
||||
/** 已导入的本地浏览器配置摘要（面板展示用） */
|
||||
export interface BrowserImportInfo {
|
||||
browser: string
|
||||
profile: string
|
||||
at: string
|
||||
cookies: boolean
|
||||
localStorage: boolean
|
||||
note?: string
|
||||
}
|
||||
|
||||
/** 浏览器面板完整状态（事件与查询共用同一结构） */
|
||||
export interface BrowserUiState {
|
||||
/** 是否存在受管浏览器会话（有标签页即视为打开） */
|
||||
open: boolean
|
||||
/** 风险告知是否已确认（首次使用门禁） */
|
||||
riskAcknowledged: boolean
|
||||
tabs: BrowserTabInfo[]
|
||||
activeTabId: string | null
|
||||
/** 最近一次 Agent 浏览器操作描述（面板底部轨迹） */
|
||||
trace: string | null
|
||||
imported?: BrowserImportInfo | null
|
||||
}
|
||||
|
||||
export type BrowserUiEvent = { type: 'state'; state: BrowserUiState }
|
||||
|
||||
/** 本地浏览器（Chromium 系）下的一个 profile */
|
||||
export interface BrowserProfileMeta {
|
||||
/** profile 目录名（Default / Profile 1 …） */
|
||||
dir: string
|
||||
/** 展示名（Preferences 里的名称或目录名） */
|
||||
name: string
|
||||
}
|
||||
|
||||
/** 检测到的可导入浏览器来源 */
|
||||
export interface BrowserSource {
|
||||
id: string
|
||||
name: string
|
||||
userDataDir: string
|
||||
profiles: BrowserProfileMeta[]
|
||||
}
|
||||
|
||||
/** 导入结果（主进程返回；cookieNote 说明跳过原因） */
|
||||
export interface BrowserImportResult {
|
||||
ok: boolean
|
||||
cookies: boolean
|
||||
localStorage: boolean
|
||||
cookieNote?: string
|
||||
message: string
|
||||
}
|
||||
|
||||
export type AgentUiEvent =
|
||||
| { type: 'session_ready'; state: SessionState }
|
||||
| { type: 'session_error'; message: string }
|
||||
|
||||
Reference in New Issue
Block a user