diff --git a/apps/desktop/src/main/browser/browser-cdp.ts b/apps/desktop/src/main/browser/browser-cdp.ts new file mode 100644 index 0000000..49de5bd --- /dev/null +++ b/apps/desktop/src/main/browser/browser-cdp.ts @@ -0,0 +1,60 @@ +/** + * CDP 命令的基础设施:超时与中止。 + * + * Electron Debugger 的 sendCommand 在目标页面卡死或 DevTools 通道异常时可能永不 + * settle;超时/中止只让调用方继续执行,底层命令后续 settle 时会被安全忽略。 + */ + +export const BROWSER_CDP_COMMAND_TIMEOUT_MS = 8_000 +export const BROWSER_OBSERVE_TIMEOUT_MS = 5_000 +export const BROWSER_OBSERVE_AX_DEPTH = 8 +export const BROWSER_OBSERVE_EXTENDED_AX_DEPTH = 16 + +/** 高预算 Observe 读取更深的 AX tree,常规观察保持紧凑避免 Chromium 长时间阻塞 */ +export function resolveBrowserObserveAxDepth(maxElements: number): number { + return maxElements > 240 ? BROWSER_OBSERVE_EXTENDED_AX_DEPTH : BROWSER_OBSERVE_AX_DEPTH +} + +export class BrowserCdpTimeoutError extends Error { + constructor(method: string, timeoutMs: number) { + super(`浏览器页面未在 ${Math.ceil(timeoutMs / 1_000)} 秒内响应 ${method},请稍后重试或重新加载页面。`) + this.name = 'BrowserCdpTimeoutError' + } +} + +/** 已发出的 DevTools 命令无法撤销;该错误只保证不再执行后续页面动作 */ +export class BrowserOperationAbortedError extends Error { + constructor() { + super('浏览器操作已停止。已发送的页面指令可能已执行,页面状态请重新观察确认。') + this.name = 'BrowserOperationAbortedError' + } +} + +export function throwIfBrowserOperationAborted(signal?: AbortSignal): void { + if (signal?.aborted) throw new BrowserOperationAbortedError() +} + +export interface CdpResponse { + [key: string]: unknown +} + +export function withBrowserCdpTimeout( + command: () => Promise, + method: string, + timeoutMs: number = BROWSER_CDP_COMMAND_TIMEOUT_MS +): Promise { + return new Promise((resolve, reject) => { + let settled = false + const settle = (callback: () => void): void => { + if (settled) return + settled = true + clearTimeout(timer) + callback() + } + const timer = setTimeout(() => settle(() => reject(new BrowserCdpTimeoutError(method, timeoutMs))), timeoutMs) + void Promise.resolve() + .then(command) + .then((value) => settle(() => resolve(value))) + .catch((error: unknown) => settle(() => reject(error))) + }) +} diff --git a/apps/desktop/src/main/browser/browser-controller.ts b/apps/desktop/src/main/browser/browser-controller.ts new file mode 100644 index 0000000..780068b --- /dev/null +++ b/apps/desktop/src/main/browser/browser-controller.ts @@ -0,0 +1,939 @@ +import { app, BrowserWindow, session as electronSession, WebContentsView, type Rectangle, type Session } from 'electron' +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import type { BrowserImportInfo, BrowserTabInfo, BrowserUiState } from '@shared/types' +import { + BrowserCdpTimeoutError, + BROWSER_OBSERVE_TIMEOUT_MS, + resolveBrowserObserveAxDepth, + withBrowserCdpTimeout, + type CdpResponse +} from './browser-cdp' +import { + assertSafeBrowserUrl, + browserObservationNameLimit, + NAMELESS_ALLOWED_ROLES, + prioritizeBrowserObservationCandidates, + resolveBrowserDestination, + resolveBrowserObserveMaxElements +} from './browser-policy' + +/** + * 内置受管浏览器控制器。 + * + * 架构(参考 Proma 的成熟实现,适配本应用单窗口/单聊天会话形态): + * - 每个标签页 = 一个原生 WebContentsView,通过 contentView.addChildView 呈现; + * 渲染层 BrowserSlot 量出占位矩形,经 IPC 同步边界(原生视图天然盖在 DOM 之上)。 + * - 页面观察/操作全部走 CDP(webContents.debugger):AX tree 快照 + ref 代际机制 + * + Input.dispatch* 真实输入事件;不在页面里执行模拟点击。 + * - 登录态隔离在 persist:aso-browser 分区;本地浏览器配置导入由 browser-profile.ts + * 直接迁移分区目录数据(Cookies/Local Storage + 密钥移植)。 + */ + +const PARTITION = 'persist:aso-browser' +const NEW_TAB_URL = 'https://www.google.com/' +const MAX_TABS = 12 +const MAX_FILL_CHARS = 10_000 +const MAX_SCRIPT_CHARS = 20_000 +const MAX_EXTRACT_CHARS = 50_000 + +interface RefEntry { + backendNodeId: number + generation: number + label: string + editable: boolean +} + +interface TabRecord { + tabId: string + view: WebContentsView + createdAt: number + url: string + title: string + loading: boolean + /** ref 表与代际号:每次观察/导航后旧 ref 失效 */ + refs: Map + generation: number + /** 同一 tab 的 Agent 操作串行化,避免 CDP 命令交错 */ + commandTail: Promise + lastLayoutRevision: number + attached: boolean +} + +interface BrowserSettings { + riskAcknowledged: boolean + imported?: BrowserImportInfo | null +} + +export interface BrowserObservation { + tabId: string + url: string + title: string + generation: number + elements: Array<{ ref: string; role: string; name: string; editable: boolean }> +} + +export interface BrowserWaitCondition { + kind: 'url' | 'text' | 'selector' + value: string +} + +function textValue(v: unknown): string { + if (v && typeof v === 'object') { + const value = (v as { value?: unknown }).value + if (typeof value === 'string') return value + } + return '' +} + +function isEditableAxNode(node: Record): boolean { + const role = textValue(node.role).toLowerCase() + if (role === 'textfield' || role === 'textbox' || role === 'searchbox') return true + const props = Array.isArray(node.properties) ? (node.properties as Array>) : [] + for (const p of props) { + if (String(p.name) === 'editable') { + const inner = p.value as { value?: unknown } | undefined + if (inner && typeof inner.value === 'boolean') return inner.value + } + } + return false +} + +/** 导航键 → CDP key 事件参数(rawKeyDown + VK code 才会触发默认行为:滚动/提交/焦点移动) */ +const KEY_EVENTS: Record = { + enter: { key: 'Enter', code: 'Enter', vk: 13 }, + tab: { key: 'Tab', code: 'Tab', vk: 9 }, + escape: { key: 'Escape', code: 'Escape', vk: 27 }, + backspace: { key: 'Backspace', code: 'Backspace', vk: 8 }, + delete: { key: 'Delete', code: 'Delete', vk: 46 }, + pageup: { key: 'PageUp', code: 'PageUp', vk: 33 }, + pagedown: { key: 'PageDown', code: 'PageDown', vk: 34 }, + home: { key: 'Home', code: 'Home', vk: 36 }, + end: { key: 'End', code: 'End', vk: 35 }, + arrowup: { key: 'ArrowUp', code: 'ArrowUp', vk: 38 }, + arrowdown: { key: 'ArrowDown', code: 'ArrowDown', vk: 40 }, + arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft', vk: 37 }, + arrowright: { key: 'ArrowRight', code: 'ArrowRight', vk: 39 }, + space: { key: ' ', code: 'Space', vk: 32 } +} + +export class BrowserController { + private owner: BrowserWindow | null = null + private browserSession: Session + private tabs = new Map() + private activeTabId: string | null = null + private agentTabId: string | null = null + private listeners = new Set<(state: BrowserUiState) => void>() + private traceText: string | null = null + private settings: BrowserSettings = { riskAcknowledged: false } + + constructor() { + this.browserSession = electronSession.fromPartition(PARTITION) + // 诚实的 UA:保留 Chromium token 保证站点兼容,去掉 Electron 标识 + this.browserSession.setUserAgent( + this.browserSession.getUserAgent().replace(/Electron\/[\d.]+/, `InkreachAgent/${app.getVersion()}`) + ) + // 网页权限请求(摄像头/定位等)默认拒绝 + this.browserSession.setPermissionRequestHandler((_wc, _permission, callback) => callback(false)) + // 下载留在受管浏览器内,落到系统下载目录(重名自动加序号) + this.browserSession.on('will-download', (_event, item) => { + const dir = app.getPath('downloads') + const base = item.getFilename() || 'download' + const dot = base.lastIndexOf('.') + const stem = dot > 0 ? base.slice(0, dot) : base + const ext = dot > 0 ? base.slice(dot) : '' + let target = join(dir, base) + for (let i = 1; existsSync(target) && i < 100; i++) { + target = join(dir, `${stem} (${i})${ext}`) + } + item.setSavePath(target) + }) + this.settings = this.loadSettings() + } + + // ---- 设置持久化 ---- + + private get settingsFile(): string { + return join(app.getPath('userData'), 'aso-browser.json') + } + + private loadSettings(): BrowserSettings { + try { + const raw = JSON.parse(readFileSync(this.settingsFile, 'utf8')) as Partial + return { + riskAcknowledged: raw.riskAcknowledged === true, + imported: raw.imported ?? null + } + } catch { + return { riskAcknowledged: false } + } + } + + private saveSettings(): void { + try { + writeFileSync(this.settingsFile, JSON.stringify(this.settings, null, 2), 'utf8') + } catch (err) { + console.error('[browser] save settings failed:', err) + } + } + + /** 配置导入完成后由 browser-profile.ts 回写摘要 */ + setImportedInfo(info: BrowserImportInfo | null): void { + this.settings.imported = info + this.saveSettings() + this.emitState() + } + + get importedInfo(): BrowserImportInfo | null { + return this.settings.imported ?? null + } + + /** 当前是否有活跃标签页(导入配置前必须没有) */ + get hasTabs(): boolean { + return this.tabs.size > 0 + } + + // ---- 窗口与状态广播 ---- + + setOwnerWindow(win: BrowserWindow): void { + this.owner = win + } + + handleWindowClosed(): void { + for (const tab of this.tabs.values()) { + try { + if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach() + } catch { + // 已销毁 + } + try { + tab.view.setVisible(false) + } catch { + // 已销毁 + } + } + this.tabs.clear() + this.activeTabId = null + this.agentTabId = null + this.owner = null + } + + onState(fn: (state: BrowserUiState) => void): () => void { + this.listeners.add(fn) + return () => this.listeners.delete(fn) + } + + private buildState(): BrowserUiState { + const tabs: BrowserTabInfo[] = [...this.tabs.values()] + .sort((a, b) => a.createdAt - b.createdAt) + .map((t) => ({ tabId: t.tabId, url: t.url, title: t.title, loading: t.loading })) + return { + open: tabs.length > 0, + riskAcknowledged: this.settings.riskAcknowledged, + tabs, + activeTabId: this.activeTabId, + trace: this.traceText, + imported: this.settings.imported ?? null + } + } + + getState(): BrowserUiState { + return this.buildState() + } + + private emitState(): void { + const state = this.buildState() + for (const fn of this.listeners) { + try { + fn(state) + } catch (err) { + console.error('[browser] state listener error:', err) + } + } + } + + private trace(tab: TabRecord, action: string, detail: string): void { + this.traceText = `[${action}] ${detail}` + void tab + this.emitState() + } + + // ---- 风险告知门禁 ---- + + acknowledgeRisk(): void { + this.settings.riskAcknowledged = true + this.saveSettings() + this.emitState() + } + + private assertRiskAcknowledged(): void { + if (!this.settings.riskAcknowledged) { + throw new Error('内置浏览器首次使用尚未确认风险告知:请在应用右侧浏览器面板中阅读并点击「我已了解,开始使用」后重试。') + } + } + + // ---- 会话与标签页 ---- + + private ensureBrowser(): TabRecord { + if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪,无法使用内置浏览器。') + if (this.tabs.size === 0) this.createTab(NEW_TAB_URL) + const active = this.activeTabId ? this.tabs.get(this.activeTabId) : null + const fallback = this.tabs.values().next().value + const tab = active ?? fallback + if (!tab) throw new Error('浏览器标签页不可用。') + this.emitState() + return tab + } + + private invalidateTabDocument(tab: TabRecord): void { + tab.generation += 1 + tab.refs.clear() + } + + private createTab(initialUrl: string, viewOptions?: Electron.WebContentsViewConstructorOptions): TabRecord { + if (!this.owner || this.owner.isDestroyed()) throw new Error('主窗口尚未就绪,无法创建浏览器标签。') + if (this.tabs.size >= MAX_TABS) throw new Error(`浏览器标签页已达上限(${MAX_TABS}),请先关闭不需要的标签。`) + const tabId = `tab-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` + const view = new WebContentsView( + viewOptions ?? { + webPreferences: { + partition: PARTITION, + nodeIntegration: false, + contextIsolation: true, + sandbox: true, + webSecurity: true + } + } + ) + const tab: TabRecord = { + tabId, + view, + createdAt: Date.now(), + url: '', + title: '新标签页', + loading: false, + refs: new Map(), + generation: 0, + commandTail: Promise.resolve(), + lastLayoutRevision: 0, + attached: false + } + view.setVisible(false) + + // popup 收编为受管同级标签(必须同步返回 webContents,故提前建好 TabRecord) + view.webContents.setWindowOpenHandler(({ url }) => { + try { + assertSafeBrowserUrl(url) + } catch { + this.trace(tab, 'popup', `已阻止不安全的新窗口: ${url.slice(0, 80)}`) + return { action: 'deny' } + } + if (this.tabs.get(tabId) !== tab || this.tabs.size >= MAX_TABS) return { action: 'deny' } + return { + action: 'allow', + outlivesOpener: true, + createWindow: (options) => { + const popup = this.createTab(url, options) + this.selectTab(popup.tabId) + this.trace(popup, 'popup', '网页打开了新标签页') + return popup.view.webContents + } + } + }) + + const updateNav = (): void => { + const wc = view.webContents + if (wc.isDestroyed()) return + try { + tab.url = wc.getURL() + tab.title = wc.getTitle() || tab.url + tab.loading = wc.isLoading() + } catch { + // 竞态时忽略 + } + this.emitState() + } + view.webContents.on('did-start-loading', () => { + tab.loading = true + this.invalidateTabDocument(tab) + this.emitState() + }) + view.webContents.on('did-stop-loading', updateNav) + view.webContents.on('page-title-updated', updateNav) + view.webContents.on('did-navigate', () => { + this.invalidateTabDocument(tab) + updateNav() + }) + view.webContents.on('did-navigate-in-page', () => { + this.invalidateTabDocument(tab) + updateNav() + }) + view.webContents.on('destroyed', () => { + this.tabs.delete(tabId) + this.removeView(view) + if (this.activeTabId === tabId) this.activeTabId = this.tabs.keys().next().value ?? null + if (this.agentTabId === tabId) this.agentTabId = this.activeTabId + this.emitState() + }) + + try { + view.webContents.debugger.attach('1.3') + } catch (err) { + console.warn('[browser] CDP attach 失败:', err) + } + this.tabs.set(tabId, tab) + if (!this.activeTabId) this.activeTabId = tabId + // popup 场景(带 viewOptions)由 Electron 的 createWindow 机制自行导航,避免双重加载 + if (initialUrl && !viewOptions) void view.webContents.loadURL(assertSafeBrowserUrl(initialUrl)).catch(() => undefined) + return tab + } + + private removeView(view: WebContentsView): void { + try { + this.owner?.contentView.removeChildView(view) + } catch { + // 窗口已销毁 + } + try { + view.webContents.close() + } catch { + // 已销毁 + } + } + + listTabs(): BrowserUiState { + return this.buildState() + } + + newTab(url?: string, byAgent = true): BrowserUiState { + this.assertRiskAcknowledged() + const tab = this.createTab(url ? resolveBrowserDestination(url) : NEW_TAB_URL) + this.activeTabId = tab.tabId + if (byAgent) this.agentTabId = tab.tabId + this.emitState() + return this.buildState() + } + + selectTab(tabId: string): BrowserUiState { + const tab = this.tabs.get(tabId) + if (!tab) throw new Error(`标签页不存在: ${tabId}`) + this.activeTabId = tabId + try { + tab.view.webContents.focus() + } catch { + // 页面可能正在加载 + } + this.emitState() + return this.buildState() + } + + closeTab(tabId?: string): BrowserUiState { + const id = tabId ?? this.activeTabId + if (!id) return this.buildState() + const tab = this.tabs.get(id) + if (!tab) return this.buildState() + try { + if (tab.view.webContents.debugger.isAttached()) tab.view.webContents.debugger.detach() + } catch { + // 已销毁 + } + this.removeView(tab.view) + this.tabs.delete(id) + if (this.activeTabId === id) this.activeTabId = this.tabs.keys().next().value ?? null + if (this.agentTabId === id) this.agentTabId = this.activeTabId + this.emitState() + return this.buildState() + } + + /** 关闭整个受管浏览器(销毁全部标签;导入配置前也会走这里) */ + close(): BrowserUiState { + for (const tabId of [...this.tabs.keys()]) this.closeTab(tabId) + this.agentTabId = null + this.traceText = null + return this.buildState() + } + + // ---- 渲染层布局同步 ---- + + applyLayout(layout: { tabId: string; visible: boolean; bounds: Rectangle; revision: number }): void { + const tab = this.tabs.get(layout.tabId) + if (!tab || !this.owner || this.owner.isDestroyed()) return + if (layout.revision <= tab.lastLayoutRevision) return + tab.lastLayoutRevision = layout.revision + try { + if (layout.visible && layout.bounds.width > 4 && layout.bounds.height > 4) { + if (!tab.attached) { + this.owner.contentView.addChildView(tab.view) + tab.attached = true + } + tab.view.setBounds(layout.bounds) + tab.view.setVisible(true) + } else { + tab.view.setVisible(false) + } + } catch (err) { + console.error('[browser] apply layout failed:', err) + } + } + + // ---- 用户导航(地址栏/按钮) ---- + + navigateDisplay(input: string): BrowserUiState { + this.assertRiskAcknowledged() + const tab = this.ensureBrowser() + void tab.view.webContents.loadURL(resolveBrowserDestination(input)).catch(() => undefined) + this.trace(tab, 'navigate', `打开 ${input.slice(0, 80)}`) + return this.buildState() + } + + goBack(): BrowserUiState { + this.assertRiskAcknowledged() + const tab = this.ensureBrowser() + if (tab.view.webContents.navigationHistory.canGoBack()) tab.view.webContents.navigationHistory.goBack() + return this.buildState() + } + + goForward(): BrowserUiState { + this.assertRiskAcknowledged() + const tab = this.ensureBrowser() + if (tab.view.webContents.navigationHistory.canGoForward()) tab.view.webContents.navigationHistory.goForward() + return this.buildState() + } + + reload(): BrowserUiState { + this.assertRiskAcknowledged() + const tab = this.ensureBrowser() + tab.view.webContents.reload() + return this.buildState() + } + + // ---- CDP 基础 ---- + + private cdp(tab: TabRecord, method: string, params?: Record, timeoutMs?: number): Promise { + const debuggerClient = tab.view.webContents.debugger + if (!debuggerClient.isAttached()) throw new Error('浏览器调试通道不可用,请重新打开该标签页。') + return withBrowserCdpTimeout(() => debuggerClient.sendCommand(method, params) as Promise, method, timeoutMs).catch( + (error: unknown) => { + if (error instanceof BrowserCdpTimeoutError) this.recoverDebugger(tab, method) + throw error + } + ) + } + + /** 超时后重连 debugger,避免一个卡死页面让后续浏览器调用全部失败 */ + private recoverDebugger(tab: TabRecord, method: string): void { + try { + const debuggerClient = tab.view.webContents.debugger + if (debuggerClient.isAttached()) debuggerClient.detach() + debuggerClient.attach('1.3') + console.warn(`[browser] ${method} 超时,已重连调试通道`) + } catch { + // 页面已关闭 + } + } + + /** Agent 的 tab 内操作串行化(用户操作不排队) */ + private runAgentOp(tab: TabRecord, fn: () => Promise): Promise { + const next = tab.commandTail.then(fn, fn) + tab.commandTail = next.catch(() => undefined) + return next + } + + private getAgentTab(tabId?: string): TabRecord { + if (tabId) { + const explicit = this.tabs.get(tabId) + if (!explicit) throw new Error('目标标签页不存在,请先 browser_list_tabs 并用 browser_new_tab/browser_select_tab 选择。') + return explicit + } + const id = this.agentTabId ?? this.activeTabId + const tab = id ? this.tabs.get(id) : null + if (tab) return tab + // Agent 首次操作时还没有任何标签页:自动开一个工作页 + return this.ensureBrowser() + } + + private assertCurrentDocument(tab: TabRecord, generation: number): void { + if (tab.generation !== generation) { + throw new Error('页面已变化(导航或重渲染),旧元素引用失效:请重新 browser_observe 后再操作。') + } + } + + private resolveRef(tab: TabRecord, ref: string): RefEntry { + const entry = tab.refs.get(ref) + if (!entry || entry.generation !== tab.generation) { + throw new Error('元素引用已失效,请先重新调用 browser_observe。') + } + return entry + } + + /** AX ref → 视口中心坐标(先滚动到可见再读 box;懒加载列表滚动后坐标会变,不能复用旧值) */ + private async centerForRef(tab: TabRecord, ref: string, generation: number): Promise<{ x: number; y: number }> { + this.assertCurrentDocument(tab, generation) + const { backendNodeId } = this.resolveRef(tab, ref) + await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId }) + this.assertCurrentDocument(tab, generation) + const box = await this.cdp(tab, 'DOM.getBoxModel', { backendNodeId }) + this.assertCurrentDocument(tab, generation) + const model = box.model as Record | undefined + const quad = Array.isArray(model?.content) ? (model?.content as unknown[]) : [] + if (quad.length < 8 || !quad.every((v) => typeof v === 'number')) { + throw new Error('目标元素当前不可点击,请重新观察页面。') + } + const n = quad as number[] + return { + x: (n[0] + n[2] + n[4] + n[6]) / 4, + y: (n[1] + n[3] + n[5] + n[7]) / 4 + } + } + + /** 点击前在页面上短暂高亮目标,用户能看清 Agent 正在操作什么 */ + private async highlightAgentTarget(tab: TabRecord, backendNodeId: number): Promise { + try { + const resolved = await this.cdp(tab, 'DOM.resolveNode', { backendNodeId }) + const objectId = (resolved.object as { objectId?: string } | undefined)?.objectId + if (!objectId) return + await this.cdp(tab, 'Runtime.callFunctionOn', { + functionDeclaration: + 'function(){const r=this.getBoundingClientRect();const d=document.createElement("div");' + + 'd.style.cssText=`position:fixed;left:${r.left-2}px;top:${r.top-2}px;width:${r.width+4}px;height:${r.height+4}px;`+' + + '"outline:2px solid #f59e0b;outline-offset:1px;pointer-events:none;z-index:2147483647;";' + + 'document.documentElement.appendChild(d);setTimeout(()=>d.remove(),700);return true}', + objectId, + returnByValue: true + }) + } catch { + // 高亮失败不影响操作 + } + } + + // ---- Agent 工具实现 ---- + + async navigate(url: string, tabId?: string): Promise { + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + this.agentTabId = tab.tabId + this.activeTabId = tab.tabId + const destination = resolveBrowserDestination(url) + await tab.view.webContents.loadURL(destination) + this.trace(tab, 'navigate', `打开 ${destination.slice(0, 80)}`) + return this.buildState() + } + + private parseAxNodes(response: CdpResponse): Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> { + const nodes = Array.isArray(response.nodes) ? (response.nodes as Array>) : [] + const candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }> = [] + for (const node of nodes) { + if (!node) continue + const backendNodeId = typeof node.backendDOMNodeId === 'number' ? node.backendDOMNodeId : 0 + const role = textValue(node.role).toLowerCase() + const name = textValue(node.name) + const editable = isEditableAxNode(node) + if (!backendNodeId || !role) continue + if (!name && !editable && !NAMELESS_ALLOWED_ROLES.has(role)) continue + candidates.push({ backendNodeId, role, name: name.slice(0, browserObservationNameLimit(role)), editable }) + } + return candidates + } + + private buildRefs(tab: TabRecord, candidates: Array<{ backendNodeId: number; role: string; name: string; editable: boolean }>): BrowserObservation['elements'] { + tab.generation += 1 + tab.refs.clear() + const elements: BrowserObservation['elements'] = [] + for (const candidate of candidates) { + const ref = `r${tab.generation}-${elements.length + 1}` + tab.refs.set(ref, { + backendNodeId: candidate.backendNodeId, + generation: tab.generation, + label: candidate.name ? `${candidate.role}「${candidate.name.slice(0, 80)}」` : candidate.role, + editable: candidate.editable + }) + elements.push({ ref, role: candidate.role, name: candidate.name, editable: candidate.editable }) + } + return elements + } + + async observe(tabId?: string, requestedMaxElements?: number): Promise { + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + this.agentTabId = tab.tabId + return this.runAgentOp(tab, async () => { + const maxElements = resolveBrowserObserveMaxElements(requestedMaxElements) + const depth = resolveBrowserObserveAxDepth(maxElements) + const response = await this.cdp( + tab, + 'Accessibility.getFullAXTree', + { depth }, + BROWSER_OBSERVE_TIMEOUT_MS + ) + const selected = prioritizeBrowserObservationCandidates(this.parseAxNodes(response), maxElements) + const elements = this.buildRefs(tab, selected) + this.trace(tab, 'observe', `读取到 ${elements.length}/${maxElements} 个元素(AX 深度 ${depth})`) + return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements } + }) + } + + async find( + query: { role?: string; name?: string; exact?: boolean; maxResults?: number }, + tabId?: string + ): Promise { + const role = query.role?.trim().toLowerCase() + const name = query.name?.trim().toLowerCase() + const maxResults = query.maxResults ?? 20 + if (!role && !name) throw new Error('语义定位至少需要 role 或 name。') + if (!Number.isInteger(maxResults) || maxResults < 1 || maxResults > 50) throw new Error('maxResults 必须是 1 到 50 的整数。') + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + this.agentTabId = tab.tabId + return this.runAgentOp(tab, async () => { + // 过滤后再套观察配额:Find 能找到被紧凑 Observe 截掉的目标 + const response = await this.cdp( + tab, + 'Accessibility.getFullAXTree', + { depth: resolveBrowserObserveAxDepth(MAX_BROWSER_OBSERVE_DEPTH_FALLBACK) }, + BROWSER_OBSERVE_TIMEOUT_MS + ) + const candidates = this.parseAxNodes(response).filter((candidate) => { + const roleMatches = !role || candidate.role === role + const candidateName = candidate.name.toLowerCase() + const nameMatches = !name || (query.exact ? candidateName === name : candidateName.includes(name)) + return roleMatches && nameMatches + }) + const elements = this.buildRefs(tab, candidates.slice(0, maxResults)) + this.trace(tab, 'find', `语义定位到 ${elements.length} 个元素(${role ?? name})`) + return { tabId: tab.tabId, url: tab.url, title: tab.title, generation: tab.generation, elements } + }) + } + + private async clickRef(tab: TabRecord, ref: string, generation: number): Promise { + const target = this.resolveRef(tab, ref) + const { x, y } = await this.centerForRef(tab, ref, generation) + this.assertCurrentDocument(tab, generation) + await this.highlightAgentTarget(tab, target.backendNodeId) + await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 }) + await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 }) + return target + } + + async click(ref: string, tabId?: string): Promise { + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + return this.runAgentOp(tab, async () => { + const generation = tab.generation + const target = await this.clickRef(tab, ref, generation) + this.trace(tab, 'click', `点击 ${target.label}`) + return this.buildState() + }) + } + + /** 点击 + 等待合并为一次串行操作(已知点击后预期状态时优先使用) */ + async act(ref: string, waitFor: BrowserWaitCondition | undefined, timeoutMs = 10_000, tabId?: string): Promise<{ state: BrowserUiState; matched: boolean }> { + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + return this.runAgentOp(tab, async () => { + const generation = tab.generation + const target = await this.clickRef(tab, ref, generation) + this.trace(tab, 'click', `点击 ${target.label}`) + if (!waitFor) return { state: this.buildState(), matched: true } + const matched = await this.waitForInternal(tab, waitFor, timeoutMs) + return { state: this.buildState(), matched } + }) + } + + async hover(ref: string, tabId?: string): Promise { + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + return this.runAgentOp(tab, async () => { + const generation = tab.generation + const target = this.resolveRef(tab, ref) + const { x, y } = await this.centerForRef(tab, ref, generation) + await this.highlightAgentTarget(tab, target.backendNodeId) + this.assertCurrentDocument(tab, generation) + await this.cdp(tab, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y }) + this.trace(tab, 'hover', `悬停 ${target.label}`) + return this.buildState() + }) + } + + async fill(ref: string, text: string, tabId?: string): Promise { + if (text.length > MAX_FILL_CHARS) throw new Error(`单次输入不能超过 ${MAX_FILL_CHARS} 个字符。`) + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + return this.runAgentOp(tab, async () => { + const generation = tab.generation + const target = this.resolveRef(tab, ref) + if (!target.editable) throw new Error('目标元素不是可编辑字段,请重新观察后选择 input、textarea 或 contenteditable。') + await this.highlightAgentTarget(tab, target.backendNodeId) + await this.cdp(tab, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backendNodeId }) + this.assertCurrentDocument(tab, generation) + await this.cdp(tab, 'DOM.focus', { backendNodeId: target.backendNodeId }) + this.assertCurrentDocument(tab, generation) + // 全选(Ctrl/Cmd+A)后 insertText 整段替换,不逐字符模拟 + const selectAllModifier = process.platform === 'darwin' ? 4 : 2 + await this.cdp(tab, 'Input.dispatchKeyEvent', { + type: 'keyDown', + key: 'a', + code: 'KeyA', + windowsVirtualKeyCode: 65, + modifiers: selectAllModifier + }) + await this.cdp(tab, 'Input.dispatchKeyEvent', { + type: 'keyUp', + key: 'a', + code: 'KeyA', + windowsVirtualKeyCode: 65, + modifiers: selectAllModifier + }) + this.assertCurrentDocument(tab, generation) + await this.cdp(tab, 'Input.insertText', { text }) + this.trace(tab, 'fill', `在 ${target.label} 输入 ${Array.from(text).length} 个字符`) + return this.buildState() + }) + } + + async press(key: string, tabId?: string): Promise { + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + return this.runAgentOp(tab, async () => { + const normalized = key.trim().toLowerCase().replace(/\s+/g, '') + const keyEvent = KEY_EVENTS[normalized] + if (keyEvent) { + const event = { key: keyEvent.key, code: keyEvent.code, windowsVirtualKeyCode: keyEvent.vk } + await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'rawKeyDown', ...event }) + await this.cdp(tab, 'Input.dispatchKeyEvent', { type: 'keyUp', ...event }) + this.trace(tab, 'press', `按下 ${keyEvent.key}`) + } else { + await this.cdp(tab, 'Input.insertText', { text: key }) + this.trace(tab, 'press', `输入 ${Array.from(key).length} 个字符`) + } + return this.buildState() + }) + } + + async scroll( + input: { deltaY?: number; position?: 'top' | 'bottom'; selector?: string }, + tabId?: string + ): Promise<{ before: { top: number; height: number }; after: { top: number; height: number }; moved: boolean }> { + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + return this.runAgentOp(tab, async () => { + const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : '' + const position = input.position === 'top' || input.position === 'bottom' ? input.position : undefined + const deltaY = Math.max(-50_000, Math.min(50_000, Math.round(input.deltaY ?? 600))) + // 固定表达式:数据由 JSON 注入,不拼接用户脚本 + const expression = + '((input)=>{const el=input.selector?document.querySelector(input.selector):(document.scrollingElement||document.documentElement);' + + 'if(!el)return {ok:false};const read=()=>({top:Math.round(el.scrollTop),height:Math.round(el.scrollHeight)});' + + 'const before=read();const top=input.position==="top"?0:input.position==="bottom"?Math.max(0,el.scrollHeight-el.clientHeight):el.scrollTop+input.deltaY;' + + 'el.scrollTo({top});const after=read();return {ok:true,before,after}})(' + + JSON.stringify({ selector, position, deltaY }) + + ')' + const response = await this.evaluateOnPage(tab, expression) + const result = (response ?? {}) as { ok?: boolean; before?: { top: number; height: number }; after?: { top: number; height: number } } + if (!result.ok || !result.before || !result.after) throw new Error('滚动目标不存在或不支持滚动,请检查 selector。') + this.trace(tab, 'scroll', `滚动页面${selector ? `(${selector.slice(0, 40)})` : ''}`) + return { before: result.before, after: result.after, moved: result.after.top !== result.before.top } + }) + } + + async extract( + input: { selector?: string; maxChars?: number }, + tabId?: string + ): Promise<{ url: string; title: string; text: string; truncated: boolean }> { + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + return this.runAgentOp(tab, async () => { + const selector = typeof input.selector === 'string' && input.selector.trim() ? input.selector.trim().slice(0, 500) : '' + const maxChars = Math.min(MAX_EXTRACT_CHARS, Math.max(500, Math.round(input.maxChars ?? MAX_EXTRACT_CHARS))) + const expression = + '((input)=>{const root=input.selector?document.querySelector(input.selector):document.body;' + + 'if(!root)return null;const text=(root.innerText||"").replace(/\\n{3,}/g,"\\n\\n");' + + 'return {text:text.slice(0,input.maxChars),truncated:text.length>input.maxChars}})(' + + JSON.stringify({ selector, maxChars }) + + ')' + const response = await this.evaluateOnPage(tab, expression) + const result = (response ?? {}) as { text?: string; truncated?: boolean } | null + if (!result || typeof result.text !== 'string') throw new Error('提取目标不存在,请检查 selector。') + this.trace(tab, 'extract', `提取正文 ${result.text.length} 字符${selector ? `(${selector.slice(0, 40)})` : ''}`) + return { url: tab.url, title: tab.title, text: result.text, truncated: result.truncated === true } + }) + } + + async waitFor(condition: BrowserWaitCondition, timeoutMs = 10_000, tabId?: string): Promise<{ matched: boolean; url: string; title: string }> { + if (!condition.value.trim()) throw new Error('等待条件不能为空。') + if (!Number.isFinite(timeoutMs) || timeoutMs < 250 || timeoutMs > 30_000) throw new Error('等待超时必须在 250 到 30000 毫秒之间。') + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + return this.runAgentOp(tab, async () => { + const matched = await this.waitForInternal(tab, condition, timeoutMs) + this.trace(tab, 'wait', `${matched ? '已满足' : '超时未满足'}${condition.kind} 条件(${condition.value.slice(0, 40)})`) + return { matched, url: tab.url, title: tab.title } + }) + } + + /** 轮询固定表达式(只读检查,不执行任意 JS) */ + private async waitForInternal(tab: TabRecord, condition: BrowserWaitCondition, timeoutMs: number): Promise { + const startedAt = Date.now() + const payload = JSON.stringify(condition).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029') + const expression = + `(() => { const condition = ${payload}; try { ` + + "if (condition.kind === 'url') return location.href.includes(condition.value); " + + "if (condition.kind === 'text') return (document.body?.innerText || '').includes(condition.value); " + + 'return !!document.querySelector(condition.value); } catch { return false; } })()' + while (Date.now() - startedAt <= timeoutMs) { + const result = await this.evaluateOnPage(tab, expression) + if (result === true) return true + await new Promise((resolve) => setTimeout(resolve, 250)) + } + return false + } + + /** Agent 逃生舱:在页面上下文执行自写 JS(有长度上限,结果 JSON 化) */ + async evaluate(script: string, tabId?: string): Promise { + if (!script.trim()) throw new Error('脚本不能为空。') + if (script.length > MAX_SCRIPT_CHARS) throw new Error(`脚本不能超过 ${MAX_SCRIPT_CHARS} 字符。`) + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + return this.runAgentOp(tab, async () => { + const result = await this.evaluateOnPage(tab, script, true) + this.trace(tab, 'js', `执行页面脚本 ${script.length} 字符`) + return result + }) + } + + private async evaluateOnPage(tab: TabRecord, expression: string, userGesture = false): Promise { + const response = await this.cdp(tab, 'Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + userGesture + }) + const exception = response.exceptionDetails as { text?: string; exception?: { description?: string } } | undefined + if (exception) { + throw new Error(`页面脚本执行出错: ${exception.exception?.description || exception.text || '未知错误'}`) + } + const result = response.result as { type?: string; value?: unknown; description?: string } | undefined + if (!result) return null + if ('value' in result) return result.value ?? null + return { type: result.type ?? 'unknown', description: result.description ?? null } + } + + async screenshot(tabId?: string): Promise<{ tabId: string; url: string; title: string; dataUrl: string }> { + this.assertRiskAcknowledged() + const tab = this.getAgentTab(tabId) + return this.runAgentOp(tab, async () => { + const image = await tab.view.webContents.capturePage() + this.trace(tab, 'screenshot', '截取当前页面') + return { tabId: tab.tabId, url: tab.url, title: tab.title, dataUrl: image.toDataURL() } + }) + } +} + +const MAX_BROWSER_OBSERVE_DEPTH_FALLBACK = 400 + +let controllerInstance: BrowserController | null = null + +/** 惰性单例:首次取用时才创建(session 相关 API 需在 app ready 后调用) */ +export function getBrowserController(): BrowserController { + if (!controllerInstance) controllerInstance = new BrowserController() + return controllerInstance +} diff --git a/apps/desktop/src/main/browser/browser-policy.ts b/apps/desktop/src/main/browser/browser-policy.ts new file mode 100644 index 0000000..30f6a7e --- /dev/null +++ b/apps/desktop/src/main/browser/browser-policy.ts @@ -0,0 +1,120 @@ +/** + * 受管浏览器的 URL 与观察策略。 + * 独立成模块便于单测与复用(渲染层不需要,主进程专用)。 + */ + +const SEARCH_URL = 'https://www.google.com/search?q=' + +/** 导航白名单:普通 web、本地开发服务与 about:blank;file:/javascript:/data: 一律拒绝 */ +export function assertSafeBrowserUrl(input: string): string { + const value = input.trim() + if (!value) throw new Error('URL 不能为空。') + if (value === 'about:blank') return value + let parsed: URL + try { + parsed = new URL(value) + } catch { + throw new Error(`无法解析的 URL: ${value}`) + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error(`不支持的地址协议(仅允许 http/https 与 about:blank): ${parsed.protocol}`) + } + return parsed.toString() +} + +function isLikelyUrl(value: string): boolean { + const candidate = `http://${value}` + try { + const url = new URL(candidate) + const host = url.hostname.toLowerCase() + // 域名 / IP(v4/v6)/ localhost 与 *.localhost 视为直达地址 + const isHost = + host === 'localhost' || + host.endsWith('.localhost') || + host.includes('.') || + host.includes(':') + // 无空格且形如域名/IP/localhost + return isHost && !/\s/.test(value) + } catch { + return false + } +} + +/** 输入归一化:URL/域名/IP/localhost 直达,普通文本走搜索 */ +export function resolveBrowserDestination(input: string): string { + const value = input.trim() + if (!value) throw new Error('地址不能为空。') + if (value === 'about:blank') return value + if (/^https?:\/\//i.test(value)) return assertSafeBrowserUrl(value) + if (isLikelyUrl(value)) return assertSafeBrowserUrl(`https://${value}`) + return SEARCH_URL + encodeURIComponent(value) +} + +// ---- Observe 观察策略 ---- + +export const DEFAULT_BROWSER_OBSERVE_MAX_ELEMENTS = 240 +export const MAX_BROWSER_OBSERVE_MAX_ELEMENTS = 400 +export const MIN_BROWSER_OBSERVE_MAX_ELEMENTS = 20 + +/** 可交互 AX role 白名单(占观察结果前 2/3 配额) */ +export const INTERACTIVE_AX_ROLES = new Set([ + 'button', + 'link', + 'textfield', + 'textbox', + 'searchbox', + 'checkbox', + 'radiobutton', + 'combobox', + 'menuitem', + 'menuitemcheckbox', + 'menuitemradio', + 'tab', + 'option', + 'switch', + 'slider' +]) + +/** 无名称也值得返回的 role(依赖 role 本身可定位) */ +export const NAMELESS_ALLOWED_ROLES = new Set(['button', 'textfield', 'textbox', 'link', 'checkbox', 'combobox']) + +export function resolveBrowserObserveMaxElements(requested?: number): number { + if (typeof requested !== 'number' || !Number.isFinite(requested)) return DEFAULT_BROWSER_OBSERVE_MAX_ELEMENTS + return Math.min(MAX_BROWSER_OBSERVE_MAX_ELEMENTS, Math.max(MIN_BROWSER_OBSERVE_MAX_ELEMENTS, Math.round(requested))) +} + +export function isInteractiveAxRole(role: string): boolean { + return INTERACTIVE_AX_ROLES.has(role.toLowerCase()) +} + +export interface BrowserObservationCandidate { + role: string + name: string + editable?: boolean +} + +/** 可交互元素优先(约 2/3 配额),其余语义上下文补足 */ +export function prioritizeBrowserObservationCandidates( + candidates: readonly T[], + maxElements: number +): T[] { + const interactiveLimit = Math.ceil(maxElements * (2 / 3)) + const interactive: T[] = [] + const context: T[] = [] + for (const candidate of candidates) { + if (interactive.length < interactiveLimit && (isInteractiveAxRole(candidate.role) || candidate.editable)) { + interactive.push(candidate) + } else { + context.push(candidate) + } + } + const contextLimit = maxElements - interactive.length + return [...interactive, ...context.slice(0, Math.max(0, contextLimit))] +} + +/** 不同 role 的可访问名称截断上限,避免长文本撑爆快照 */ +export function browserObservationNameLimit(role: string): number { + if (role === 'link' || role === 'button' || role === 'menuitem') return 80 + if (role === 'heading') return 120 + return 160 +} diff --git a/apps/desktop/src/main/browser/browser-profile.ts b/apps/desktop/src/main/browser/browser-profile.ts new file mode 100644 index 0000000..71e062a --- /dev/null +++ b/apps/desktop/src/main/browser/browser-profile.ts @@ -0,0 +1,256 @@ +import { app } from 'electron' +import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' +import type { BrowserImportResult, BrowserProfileMeta, BrowserSource } from '@shared/types' +import { getBrowserController } from './browser-controller' + +/** + * 本地浏览器配置导入(复用各网站登录态)。 + * + * 实现思路(零解密依赖,纯文件迁移): + * - Local Storage / IndexedDB 在 Chromium 里是明文 leveldb,直接拷入受管分区即可生效。 + * - Cookies 用 AES-GCM 加密,密钥经 OS 加密后存于浏览器根目录 Local State + * (os_crypt.encrypted_key)。Windows 上该密钥由 DPAPI(用户级)包裹——同一 + * Windows 用户的任意进程都能解开,因此把这份密钥「移植」进本应用的 Local State, + * 本应用内的 Chromium 即可解密拷贝过来的 Cookies(Chrome 127+ 的 app-bound + * 加密 V20 除外,会明确提示不支持)。macOS/Linux 的密钥绑定钥匙串/KWallet, + * Cookies 无法直接迁移,仅迁移 Local Storage。 + */ + +/** Chromium 系浏览器的用户数据根目录(按平台) */ +const SOURCE_ROOTS: Array<{ id: string; name: string; win: string; mac: string; linux: string }> = [ + { + id: 'chrome', + name: 'Google Chrome', + win: join(process.env.LOCALAPPDATA ?? '', 'Google', 'Chrome', 'User Data'), + mac: join(homedir(), 'Library', 'Application Support', 'Google', 'Chrome'), + linux: join(homedir(), '.config', 'google-chrome') + }, + { + id: 'edge', + name: 'Microsoft Edge', + win: join(process.env.LOCALAPPDATA ?? '', 'Microsoft', 'Edge', 'User Data'), + mac: join(homedir(), 'Library', 'Application Support', 'Microsoft Edge'), + linux: join(homedir(), '.config', 'microsoft-edge') + }, + { + id: 'brave', + name: 'Brave', + win: join(process.env.LOCALAPPDATA ?? '', 'BraveSoftware', 'Brave-Browser', 'User Data'), + mac: join(homedir(), 'Library', 'Application Support', 'BraveSoftware', 'Brave-Browser'), + linux: join(homedir(), '.config', 'BraveSoftware', 'Brave-Browser') + }, + { + id: 'chromium', + name: 'Chromium', + win: join(process.env.LOCALAPPDATA ?? '', 'Chromium', 'User Data'), + mac: join(homedir(), 'Library', 'Application Support', 'Chromium'), + linux: join(homedir(), '.config', 'chromium') + } +] + +/** 受管浏览器分区目录(persist:aso-browser 的落盘位置) */ +function partitionDir(): string { + return join(app.getPath('userData'), 'Partitions', 'aso-browser') +} + +function sourceRoot(id: string): string { + const source = SOURCE_ROOTS.find((s) => s.id === id) + if (!source) return '' + return process.platform === 'win32' ? source.win : process.platform === 'darwin' ? source.mac : source.linux +} + +/** 读取 profile 的展示名(Preferences 的 profile.name,失败时用目录名) */ +function readProfileName(profileDir: string): string { + try { + const file = join(profileDir, 'Preferences') + if (statSync(file).size > 8 * 1024 * 1024) return profileDir.split(/[\\/]/).pop() || profileDir + const raw = JSON.parse(readFileSync(file, 'utf8')) as { profile?: { name?: string } } + return raw.profile?.name?.trim() || profileDir.split(/[\\/]/).pop() || profileDir + } catch { + return profileDir.split(/[\\/]/).pop() || profileDir + } +} + +/** 检测本机可导入的浏览器与 profile 列表 */ +export function detectBrowserSources(): BrowserSource[] { + const sources: BrowserSource[] = [] + for (const source of SOURCE_ROOTS) { + const root = sourceRoot(source.id) + if (!root || !existsSync(root)) continue + const profiles: BrowserProfileMeta[] = [] + try { + for (const entry of readdirSync(root, { withFileTypes: true })) { + if (!entry.isDirectory()) continue + if (entry.name.startsWith('.') || entry.name === 'System Profile' || entry.name === 'Guest Profile') continue + const profileDir = join(root, entry.name) + if (!existsSync(join(profileDir, 'Preferences'))) continue + profiles.push({ dir: entry.name, name: readProfileName(profileDir) }) + } + } catch { + // 目录不可读则跳过该浏览器 + continue + } + if (profiles.length > 0) { + sources.push({ id: source.id, name: source.name, userDataDir: root, profiles }) + } + } + return sources +} + +function copyFileSafe(from: string, to: string): void { + try { + copyFileSync(from, to) + } catch (err) { + const code = (err as NodeJS.ErrnoException).code + if (code === 'EBUSY' || code === 'EPERM' || code === 'EACCES') { + throw new Error('源浏览器文件被占用:请完全退出对应的浏览器(含后台进程)后重试。') + } + throw err + } +} + +/** 拷贝 leveldb 目录(跳过 LOCK/LOG 运行时文件),目标已存在时先清空 */ +function copyLevelDbDir(from: string, to: string): boolean { + if (!existsSync(from)) return false + const entries = readdirSync(from, { withFileTypes: true }) + const files = entries.filter((e) => e.isFile() && e.name !== 'LOCK' && !e.name.startsWith('LOG')) + if (files.length === 0) return false + rmSync(to, { recursive: true, force: true }) + mkdirSync(to, { recursive: true }) + for (const file of files) { + copyFileSafe(join(from, file.name), join(to, file.name)) + } + return true +} + +interface SourceKeyInfo { + key: string | null + /** app-bound(V20)等不可移植场景的说明 */ + note?: string +} + +/** + * 读取源浏览器 Local State 里的 os_crypt.encrypted_key 并判断可移植性。 + * Windows:前缀 DPAPI 可移植;V20 为 app-bound 加密(Chrome/Edge 127+), + * 其他进程无法解开,明确降级。macOS/Linux:密钥在系统钥匙串/KWallet,跳过。 + */ +function readSourceKey(userDataDir: string): SourceKeyInfo { + if (process.platform !== 'win32') { + return { key: null, note: '当前平台的浏览器 Cookie 密钥绑定系统钥匙串,暂不支持迁移 Cookie(LocalStorage 登录态仍会导入)。' } + } + try { + const raw = JSON.parse(readFileSync(join(userDataDir, 'Local State'), 'utf8')) as { + os_crypt?: { encrypted_key?: string } + } + const encoded = raw.os_crypt?.encrypted_key + if (!encoded) return { key: null, note: '未找到源浏览器 Cookie 密钥,跳过 Cookie 迁移。' } + const bytes = Buffer.from(encoded, 'base64') + if (bytes.subarray(0, 3).toString('utf8') === 'V20') { + return { key: null, note: '该浏览器使用了应用绑定加密(app-bound encryption),无法第三方迁移 Cookie;已导入 Local Storage,首次访问各站点仍需登录一次。' } + } + if (bytes.subarray(0, 5).toString('utf8') !== 'DPAPI') { + return { key: null, note: '源浏览器 Cookie 密钥格式不受支持,跳过 Cookie 迁移。' } + } + return { key: encoded } + } catch { + return { key: null, note: '读取源浏览器 Local State 失败,跳过 Cookie 迁移。' } + } +} + +/** 把源密钥移植进本应用根 Local State(首次会备份原文件) */ +function transplantKey(sourceKey: string): void { + const appLocalState = join(app.getPath('userData'), 'Local State') + const backup = `${appLocalState}.aso-backup` + let doc: Record = {} + if (existsSync(appLocalState)) { + try { + doc = JSON.parse(readFileSync(appLocalState, 'utf8')) as Record + if (!existsSync(backup)) copyFileSync(appLocalState, backup) + } catch { + doc = {} + } + } + const osCrypt = (doc.os_crypt ?? {}) as Record + osCrypt.encrypted_key = sourceKey + doc.os_crypt = osCrypt + writeFileSync(appLocalState, JSON.stringify(doc, null, 2), 'utf8') +} + +/** 导入指定浏览器的 profile 配置到受管浏览器分区 */ +export function importBrowserProfile(sourceId: string, profileDir: string): BrowserImportResult { + const browser = getBrowserController() + if (browser.hasTabs) { + throw new Error('请先关闭内置浏览器的所有标签页,再导入浏览器配置。') + } + const root = sourceRoot(sourceId) + if (!root || !existsSync(root)) throw new Error('未找到该浏览器的用户数据目录。') + // 只接受该根目录下的直接子目录名,拒绝任意路径注入 + const dirName = profileDir.split(/[\\/]/).pop() ?? '' + const sourceProfile = join(root, dirName) + if (dirName !== profileDir || !existsSync(join(sourceProfile, 'Preferences'))) { + throw new Error('profile 目录不存在。') + } + const sourceName = SOURCE_ROOTS.find((s) => s.id === sourceId)?.name ?? sourceId + const target = partitionDir() + + // 1) Local Storage(明文 leveldb,直接迁移) + let localStorage = false + try { + localStorage = copyLevelDbDir(join(sourceProfile, 'Local Storage', 'leveldb'), join(target, 'Local Storage', 'leveldb')) + } catch (err) { + throw new Error(err instanceof Error ? err.message : `Local Storage 迁移失败: ${String(err)}`) + } + + // 2) Cookies + 密钥移植 + let cookies = false + let cookieNote: string | undefined + const keyInfo = readSourceKey(root) + const cookieDb = existsSync(join(sourceProfile, 'Network', 'Cookies')) + ? join(sourceProfile, 'Network', 'Cookies') + : existsSync(join(sourceProfile, 'Cookies')) + ? join(sourceProfile, 'Cookies') + : null + if (keyInfo.key && cookieDb) { + try { + transplantKey(keyInfo.key) + mkdirSync(join(target, 'Network'), { recursive: true }) + copyFileSafe(cookieDb, join(target, 'Network', 'Cookies')) + for (const suffix of ['-wal', '-shm']) { + if (existsSync(`${cookieDb}${suffix}`)) copyFileSafe(`${cookieDb}${suffix}`, join(target, 'Network', `Cookies${suffix}`)) + } + cookies = true + } catch (err) { + cookieNote = err instanceof Error ? err.message : 'Cookie 迁移失败。' + } + } else { + cookieNote = keyInfo.note ?? '源浏览器没有可迁移的 Cookie 数据库。' + } + + const parts: string[] = [] + if (cookies) parts.push('Cookie(站点登录态)') + if (localStorage) parts.push('LocalStorage') + if (!cookies && !localStorage) { + return { ok: false, cookies, localStorage, cookieNote, message: `未在 ${sourceName}(${dirName})中找到可迁移的登录数据。` } + } + + browser.setImportedInfo({ + browser: sourceName, + profile: readProfileName(sourceProfile), + at: new Date().toISOString(), + cookies, + localStorage, + note: cookieNote + }) + + return { + ok: true, + cookies, + localStorage, + cookieNote, + message: `已从 ${sourceName} 导入 ${parts.join(' + ')}。${cookieNote ?? ''}${ + cookies ? '若登录态未生效,重启应用后重试。' : '' + }` + } +} diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 3b21028..0e0a19e 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -2,6 +2,7 @@ import { app, BrowserWindow, ipcMain, Menu, shell } from 'electron' import { existsSync } from 'node:fs' import { join } from 'node:path' import { registerIpc } from './ipc' +import { getBrowserController } from './browser/browser-controller' process.env.APP_ROOT = join(__dirname, '..') @@ -71,9 +72,13 @@ async function createWindow(): Promise { }) }) - // 窗口关闭时未决审批一律拒绝,避免 agent 永久挂起 + // 受管浏览器的原生 WebContentsView 挂在主窗口 contentView 上 + getBrowserController().setOwnerWindow(win) + + // 窗口关闭时未决审批一律拒绝,避免 agent 永久挂起;受管浏览器同步清理 win.on('closed', () => { agentService.approvals.settleAll(false) + getBrowserController().handleWindowClosed() }) // 外部链接交给系统默认浏览器,不在应用内开新窗口 diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index 99f1a7a..466dc39 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -2,6 +2,8 @@ import { BrowserWindow, dialog, ipcMain, shell } from 'electron' import { mkdirSync } from 'node:fs' import { userInfo } from 'node:os' import { AgentService } from './pi/agent-service' +import { getBrowserController } from './browser/browser-controller' +import { detectBrowserSources, importBrowserProfile } from './browser/browser-profile' /** * 注册 IPC 通道并把 agent 事件转发到所有渲染窗口。 @@ -257,5 +259,70 @@ export function registerIpc(): AgentService { return service.pickAttachments(win ?? undefined) }) + // ---- 内置受管浏览器 ---- + const browser = getBrowserController() + // 面板状态广播(与 agent:event 同样转发到所有窗口) + browser.onState((state) => { + for (const win of BrowserWindow.getAllWindows()) { + if (!win.isDestroyed()) win.webContents.send('browser:event', { type: 'state', state }) + } + }) + + const browserPass = (fn: () => T): T => { + try { + return fn() + } catch (err) { + throw new Error(err instanceof Error ? err.message : String(err)) + } + } + + ipcMain.handle('browser:getState', () => browser.getState()) + ipcMain.handle('browser:navigate', (_e, input: unknown) => + browserPass(() => browser.navigateDisplay(typeof input === 'string' ? input : '')) + ) + ipcMain.handle('browser:goBack', () => browserPass(() => browser.goBack())) + ipcMain.handle('browser:goForward', () => browserPass(() => browser.goForward())) + ipcMain.handle('browser:reload', () => browserPass(() => browser.reload())) + ipcMain.handle('browser:newTab', (_e, url: unknown) => + browserPass(() => browser.newTab(typeof url === 'string' && url ? url : undefined, false)) + ) + ipcMain.handle('browser:selectTab', (_e, tabId: unknown) => + browserPass(() => browser.selectTab(typeof tabId === 'string' ? tabId : '')) + ) + ipcMain.handle('browser:closeTab', (_e, tabId: unknown) => + browserPass(() => browser.closeTab(typeof tabId === 'string' ? tabId : undefined)) + ) + ipcMain.handle('browser:close', () => browserPass(() => browser.close())) + ipcMain.handle('browser:acknowledgeRisk', () => browserPass(() => browser.acknowledgeRisk())) + // 渲染层 BrowserSlot 量出的占位矩形(原生 WebContentsView 对齐到 DOM 布局) + ipcMain.handle('browser:layout', (_e, layout: unknown) => { + const l = layout as { tabId?: unknown; visible?: unknown; revision?: unknown; bounds?: unknown } + const { tabId, visible, revision } = l + const bounds = l.bounds as { x?: unknown; y?: unknown; width?: unknown; height?: unknown } | undefined + const x = bounds?.x + const y = bounds?.y + const width = bounds?.width + const height = bounds?.height + if ( + typeof tabId !== 'string' || + typeof visible !== 'boolean' || + typeof revision !== 'number' || + typeof x !== 'number' || + typeof y !== 'number' || + typeof width !== 'number' || + typeof height !== 'number' + ) { + throw new Error('invalid browser layout') + } + return browserPass(() => browser.applyLayout({ tabId, visible, revision, bounds: { x, y, width, height } })) + }) + + // ---- 本地浏览器配置导入 ---- + ipcMain.handle('browser:detectSources', () => browserPass(() => detectBrowserSources())) + ipcMain.handle('browser:importProfile', (_e, sourceId: unknown, profileDir: unknown) => { + if (typeof sourceId !== 'string' || typeof profileDir !== 'string') throw new Error('invalid import source') + return browserPass(() => importBrowserProfile(sourceId, profileDir)) + }) + return service } diff --git a/apps/desktop/src/main/pi/agent-service.ts b/apps/desktop/src/main/pi/agent-service.ts index e4d1fb0..6665c5a 100644 --- a/apps/desktop/src/main/pi/agent-service.ts +++ b/apps/desktop/src/main/pi/agent-service.ts @@ -16,6 +16,7 @@ import { } from '@earendil-works/pi-coding-agent' import { ApprovalService, createApprovalExtension } from './approval' import { demoTools, TOOLS_REQUIRING_APPROVAL } from '../tools/demo-tools' +import { browserTools } from '../tools/browser-tools' import type { AgentUiEvent, AppSettings, @@ -269,7 +270,7 @@ export class AgentService { services, sessionManager, sessionStartEvent, - customTools: demoTools + customTools: [...demoTools, ...browserTools] }) return { ...result, services, diagnostics: services.diagnostics } } diff --git a/apps/desktop/src/main/tools/browser-tools.ts b/apps/desktop/src/main/tools/browser-tools.ts new file mode 100644 index 0000000..54e6fc7 --- /dev/null +++ b/apps/desktop/src/main/tools/browser-tools.ts @@ -0,0 +1,292 @@ +import { defineTool } from '@earendil-works/pi-coding-agent' +import { Type } from '@earendil-works/pi-ai' +import { getBrowserController } from '../browser/browser-controller' + +/** + * 内置受管浏览器工具(进程内一等工具,非 MCP)。 + * + * 使用范式(与 Playwright MCP / Proma 一致的 AX 快照 + ref 体系): + * 1. browser_observe / browser_find 拿到带 ref 的可交互元素快照; + * 2. 用 ref 调 click / fill / act 等操作;页面变化后旧 ref 失效,必须重新观察; + * 3. 语义结构足够时优先 observe,需要视觉验证时才 browser_screenshot。 + */ + +const TAB_ID_PARAM = Type.Optional( + Type.String({ description: '可选,目标标签页 ID。默认为 Agent 工作标签页(与用户可见面板同步)。' }) +) + +export const browserTools = [ + defineTool({ + name: 'browser_navigate', + label: '打开网页', + description: + '在内置受管浏览器中打开 URL 或搜索词。完整 URL、裸域名、localhost 与 IP 直达;普通文本用 Google 搜索。支持 about:blank 空白页。下载与弹窗会留在受管浏览器内。', + promptSnippet: 'browser_navigate: 打开网页/搜索(URL 或关键词)', + parameters: Type.Object({ + url: Type.String({ description: 'URL、域名或搜索关键词' }), + tabId: TAB_ID_PARAM + }), + async execute(_toolCallId, params) { + const state = await getBrowserController().navigate(params.url, params.tabId) + return { content: [{ type: 'text', text: JSON.stringify(state.tabs.find((t) => t.tabId === state.activeTabId) ?? state) }], details: state } + } + }), + + defineTool({ + name: 'browser_observe', + label: '观察网页', + description: + '读取当前页面的 URL、标题与紧凑可访问性快照(元素带 ref,标注 editable)。默认返回 240 个元素(约 2/3 可交互优先)。每次 observe/find 或页面导航后旧 ref 全部失效,操作必须使用最新一次观察结果中的 ref。', + promptSnippet: 'browser_observe: 读页面结构拿 ref(先观察再操作)', + parameters: Type.Object({ + maxElements: Type.Optional( + Type.Number({ minimum: 20, maximum: 400, description: '返回元素上限,默认 240;仅当目标不在快照中时才提高到 400' }) + ), + tabId: TAB_ID_PARAM + }), + async execute(_toolCallId, params) { + const observation = await getBrowserController().observe(params.tabId, params.maxElements) + return { content: [{ type: 'text', text: JSON.stringify(observation) }], details: observation } + } + }), + + defineTool({ + name: 'browser_find', + label: '语义定位元素', + description: + '按可访问性 role 和/或 name 定位少量新 ref,不返回整页快照。适合 observe 结果过大或目标被截掉时使用。与 observe 一样会作废同标签页的全部旧 ref。', + promptSnippet: 'browser_find: 按 role/name 精准找元素', + parameters: Type.Object({ + role: Type.Optional(Type.String({ description: '可访问性 role,如 button/textbox/link/checkbox' })), + name: Type.Optional(Type.String({ description: '可访问性名称查询' })), + exact: Type.Optional(Type.Boolean({ description: '精确匹配 name(默认包含匹配)' })), + maxResults: Type.Optional(Type.Number({ minimum: 1, maximum: 50, description: '最多返回条数,默认 20' })), + tabId: TAB_ID_PARAM + }), + async execute(_toolCallId, params) { + const result = await getBrowserController().find( + { role: params.role, name: params.name, exact: params.exact, maxResults: params.maxResults }, + params.tabId + ) + return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result } + } + }), + + defineTool({ + name: 'browser_click', + label: '点击元素', + description: '点击最新 observe/find 结果中的指定 ref。页面会短暂高亮目标。点击导致的重渲染会让旧 ref 失效,操作后需重新观察。', + promptSnippet: 'browser_click: 点击 ref', + parameters: Type.Object({ ref: Type.String({ description: '元素引用(来自 observe/find)' }), tabId: TAB_ID_PARAM }), + async execute(_toolCallId, params) { + const state = await getBrowserController().click(params.ref, params.tabId) + return { content: [{ type: 'text', text: JSON.stringify({ clicked: params.ref, url: state.tabs.find((t) => t.tabId === state.activeTabId)?.url }) }], details: state } + } + }), + + defineTool({ + name: 'browser_act', + label: '点击并等待', + description: + '点击 ref 后在同一次串行操作中等待一个固定条件(URL 片段/可见文本/CSS selector)。已知点击后的预期状态时优先用它,避免 click+wait 两次调用。', + promptSnippet: 'browser_act: 点击 + 等待条件(原子操作)', + parameters: Type.Object({ + ref: Type.String({ description: '元素引用(来自 observe/find)' }), + waitFor: Type.Optional( + Type.Object({ + kind: Type.Union([Type.Literal('url'), Type.Literal('text'), Type.Literal('selector')]), + value: Type.String({ description: 'URL 片段 / 可见文本 / CSS selector' }) + }) + ), + timeoutMs: Type.Optional(Type.Number({ minimum: 250, maximum: 30000, description: '等待超时毫秒,默认 10000' })), + tabId: TAB_ID_PARAM + }), + async execute(_toolCallId, params) { + const result = await getBrowserController().act(params.ref, params.waitFor, params.timeoutMs, params.tabId) + return { content: [{ type: 'text', text: JSON.stringify({ clicked: params.ref, matched: result.matched }) }], details: result.state } + } + }), + + defineTool({ + name: 'browser_fill', + label: '填写输入框', + description: + '整体替换指定 ref 的 input/textarea/contenteditable 内容(先全选再插入,支持空格、标点、Unicode 与换行)。完整消息、搜索词、多行文本都优先用它。', + promptSnippet: 'browser_fill: 替换输入框内容', + parameters: Type.Object({ + ref: Type.String({ description: '可编辑字段的 ref(observe 结果中 editable=true)' }), + text: Type.String({ description: '要输入的完整文本,≤10000 字符' }), + tabId: TAB_ID_PARAM + }), + async execute(_toolCallId, params) { + const state = await getBrowserController().fill(params.ref, params.text, params.tabId) + return { content: [{ type: 'text', text: JSON.stringify({ filled: params.ref, chars: Array.from(params.text).length }) }], details: state } + } + }), + + defineTool({ + name: 'browser_press', + label: '按键', + description: + '按导航键(Enter/Tab/Escape/方向键/PageDown 等)或向已聚焦的编辑器一次性插入完整文本。字段有 ref 时优先 browser_fill。', + promptSnippet: 'browser_press: Enter/Tab 等按键或向焦点插入文本', + parameters: Type.Object({ key: Type.String({ description: '导航键名,或要插入的完整文本' }), tabId: TAB_ID_PARAM }), + async execute(_toolCallId, params) { + const state = await getBrowserController().press(params.key, params.tabId) + return { content: [{ type: 'text', text: JSON.stringify({ pressed: params.key }) }], details: state } + } + }), + + defineTool({ + name: 'browser_hover', + label: '悬停元素', + description: '把指针移到指定 ref 上(真实 mouseMoved 事件),用于展开悬浮菜单/提示,随后需重新观察再点击新内容。', + promptSnippet: 'browser_hover: 悬停展开菜单', + parameters: Type.Object({ ref: Type.String({ description: '元素引用' }), tabId: TAB_ID_PARAM }), + async execute(_toolCallId, params) { + const state = await getBrowserController().hover(params.ref, params.tabId) + return { content: [{ type: 'text', text: JSON.stringify({ hovered: params.ref }) }], details: state } + } + }), + + defineTool({ + name: 'browser_scroll', + label: '滚动页面', + description: + '滚动页面或 CSS selector 指定的内部滚动容器(SPA 信息流常在内部容器里,导航键滚不动)。返回前后 scrollTop 指标,用 moved 核验是否真的滚动。', + promptSnippet: 'browser_scroll: 滚动页面/内部容器', + parameters: Type.Object({ + deltaY: Type.Optional(Type.Number({ description: '滚动像素,正向下负向上,默认 600' })), + position: Type.Optional(Type.Union([Type.Literal('top'), Type.Literal('bottom')], { description: '直接滚到顶部/底部(优先于 deltaY)' })), + selector: Type.Optional(Type.String({ description: '内部滚动容器的 CSS selector,缺省滚主文档' })), + tabId: TAB_ID_PARAM + }), + async execute(_toolCallId, params) { + const result = await getBrowserController().scroll( + { deltaY: params.deltaY, position: params.position, selector: params.selector }, + params.tabId + ) + return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result } + } + }), + + defineTool({ + name: 'browser_extract', + label: '提取页面文本', + description: + '从页面正文或 selector 区域提取受限长度的纯文本,替代为读内容而执行 JS。优先传 selector 限定正文/列表区域,避免导航栏与页脚噪声。', + promptSnippet: 'browser_extract: 提取正文/区域文本', + parameters: Type.Object({ + selector: Type.Optional(Type.String({ description: '限定提取区域的 CSS selector,缺省取 body' })), + maxChars: Type.Optional(Type.Number({ description: '最多提取字符数,默认 50000' })), + tabId: TAB_ID_PARAM + }), + async execute(_toolCallId, params) { + const result = await getBrowserController().extract({ selector: params.selector, maxChars: params.maxChars }, params.tabId) + return { content: [{ type: 'text', text: result.text }], details: { url: result.url, title: result.title, truncated: result.truncated } } + } + }), + + defineTool({ + name: 'browser_wait_for', + label: '等待页面状态', + description: '等待一个固定条件:URL 片段、可见文本或 CSS selector。超时返回 matched=false,不执行任意 JavaScript。', + promptSnippet: 'browser_wait_for: 等待 url/text/selector 条件', + parameters: Type.Object({ + kind: Type.Union([Type.Literal('url'), Type.Literal('text'), Type.Literal('selector')]), + value: Type.String({ minLength: 1, maxLength: 2000, description: 'URL 片段 / 可见文本 / CSS selector' }), + timeoutMs: Type.Optional(Type.Number({ minimum: 250, maximum: 30000, description: '超时毫秒,默认 10000' })), + tabId: TAB_ID_PARAM + }), + async execute(_toolCallId, params) { + const result = await getBrowserController().waitFor({ kind: params.kind, value: params.value }, params.timeoutMs, params.tabId) + return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result } + } + }), + + defineTool({ + name: 'browser_screenshot', + label: '截图', + description: '截取当前页面。语义结构(observe)足够时不要截图;需要视觉验证、布局或渲染证据时使用。', + promptSnippet: 'browser_screenshot: 视觉验证用', + parameters: Type.Object({ tabId: TAB_ID_PARAM }), + async execute(_toolCallId, params) { + const shot = await getBrowserController().screenshot(params.tabId) + return { + content: [ + { + type: 'image' as const, + data: shot.dataUrl.replace(/^data:image\/\w+;base64,/, ''), + mimeType: 'image/png' + }, + { type: 'text', text: JSON.stringify({ url: shot.url, title: shot.title }) } + ], + details: { url: shot.url, title: shot.title } + } + } + }), + + defineTool({ + name: 'browser_list_tabs', + label: '列出标签页', + description: '列出受管浏览器的全部标签页(tabId/URL/标题/加载状态)与当前激活页。操作其他标签页时先调它拿 tabId。', + promptSnippet: 'browser_list_tabs: 查看标签页列表', + parameters: Type.Object({}), + async execute() { + const state = getBrowserController().listTabs() + return { content: [{ type: 'text', text: JSON.stringify({ tabs: state.tabs, activeTabId: state.activeTabId }) }], details: state } + } + }), + + defineTool({ + name: 'browser_new_tab', + label: '新建标签页', + description: '新建 Agent 工作标签页并激活到用户可见的浏览器面板;完成后不再需要的标签页应主动 browser_close_tab。', + promptSnippet: 'browser_new_tab: 新建工作标签页', + parameters: Type.Object({ + url: Type.Optional(Type.String({ description: '初始地址(URL/域名/搜索词),缺省开空白起始页' })) + }), + async execute(_toolCallId, params) { + const state = await getBrowserController().newTab(params.url, true) + return { content: [{ type: 'text', text: JSON.stringify({ tabId: state.activeTabId, tabs: state.tabs.length }) }], details: state } + } + }), + + defineTool({ + name: 'browser_select_tab', + label: '切换标签页', + description: '把指定标签页设为 Agent 工作页,并同步激活到用户可见面板。后续不带 tabId 的操作都作用于它。', + promptSnippet: 'browser_select_tab: 切换工作标签页', + parameters: Type.Object({ tabId: Type.String({ description: 'browser_list_tabs 返回的标签页 ID' }) }), + async execute(_toolCallId, params) { + const state = await getBrowserController().selectTab(params.tabId) + return { content: [{ type: 'text', text: JSON.stringify({ activeTabId: state.activeTabId }) }], details: state } + } + }), + + defineTool({ + name: 'browser_close_tab', + label: '关闭标签页', + description: '关闭指定标签页(缺省关闭当前工作页)。绝不关闭用户手动打开的标签页。', + promptSnippet: 'browser_close_tab: 关闭工作标签页', + parameters: Type.Object({ tabId: Type.Optional(Type.String({ description: '要关闭的标签页 ID,缺省当前工作页' })) }), + async execute(_toolCallId, params) { + const state = await getBrowserController().closeTab(params.tabId) + return { content: [{ type: 'text', text: JSON.stringify({ closed: params.tabId ?? 'current', remaining: state.tabs.length }) }], details: state } + } + }), + + defineTool({ + name: 'browser_close', + label: '关闭浏览器', + description: '关闭整个内置受管浏览器会话(销毁全部标签页)。任务结束、不再需要网页时主动调用。', + promptSnippet: 'browser_close: 关闭受管浏览器', + parameters: Type.Object({}), + async execute() { + const state = getBrowserController().close() + return { content: [{ type: 'text', text: JSON.stringify({ closed: true }) }], details: state } + } + }) +] + +export const browserToolNames = new Set(browserTools.map((tool) => tool.name)) diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index e93c747..dcc1ad1 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -3,6 +3,10 @@ import type { AgentUiEvent, AppSettings, AttachmentInfo, + BrowserImportResult, + BrowserSource, + BrowserUiEvent, + BrowserUiState, ChatResources, CustomProviderConfig, ModelCatalog, @@ -72,6 +76,33 @@ const api = { return () => { ipcRenderer.removeListener('agent:event', listener) } + }, + // ---- 内置受管浏览器 ---- + browserGetState: (): Promise => ipcRenderer.invoke('browser:getState'), + browserNavigate: (input: string): Promise => ipcRenderer.invoke('browser:navigate', input), + browserGoBack: (): Promise => ipcRenderer.invoke('browser:goBack'), + browserGoForward: (): Promise => ipcRenderer.invoke('browser:goForward'), + browserReload: (): Promise => ipcRenderer.invoke('browser:reload'), + browserNewTab: (url?: string): Promise => ipcRenderer.invoke('browser:newTab', url), + browserSelectTab: (tabId: string): Promise => ipcRenderer.invoke('browser:selectTab', tabId), + browserCloseTab: (tabId?: string): Promise => ipcRenderer.invoke('browser:closeTab', tabId), + browserClose: (): Promise => ipcRenderer.invoke('browser:close'), + browserAcknowledgeRisk: (): Promise => ipcRenderer.invoke('browser:acknowledgeRisk'), + browserSetLayout: (layout: { + tabId: string + visible: boolean + revision: number + bounds: { x: number; y: number; width: number; height: number } + }): Promise => ipcRenderer.invoke('browser:layout', layout), + browserDetectSources: (): Promise => ipcRenderer.invoke('browser:detectSources'), + browserImportProfile: (sourceId: string, profileDir: string): Promise => + ipcRenderer.invoke('browser:importProfile', sourceId, profileDir), + onBrowserEvent: (callback: (event: BrowserUiEvent) => void): (() => void) => { + const listener = (_e: Electron.IpcRendererEvent, event: BrowserUiEvent): void => callback(event) + ipcRenderer.on('browser:event', listener) + return () => { + ipcRenderer.removeListener('browser:event', listener) + } } } diff --git a/apps/desktop/src/renderer/src/App.tsx b/apps/desktop/src/renderer/src/App.tsx index 98726c4..ade4d5d 100644 --- a/apps/desktop/src/renderer/src/App.tsx +++ b/apps/desktop/src/renderer/src/App.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState } from 'react' import type { JSX } from 'react' import { useAgent } from './hooks/useAgent' +import { useBrowser } from './hooks/useBrowser' import { SidebarNav } from './components/SidebarNav' import { TitleBar } from './components/TitleBar' import { ChatView } from './components/ChatView' @@ -8,6 +9,7 @@ import { BottomSlot } from './components/BottomSlot' import { Composer } from './components/Composer' import { SettingsModal } from './components/SettingsModal' import { Search } from './components/Search' +import { BrowserPanel } from './components/browser/BrowserPanel' import { applyThemeColor } from './lib/theme' export default function App(): JSX.Element { @@ -31,6 +33,8 @@ export default function App(): JSX.Element { const prefillRef = useRef(null) const [settingsOpen, setSettingsOpen] = useState(false) const [searchOpen, setSearchOpen] = useState(false) + const [browserOpen, setBrowserOpen] = useState(false) + const browser = useBrowser() // 启动时应用持久化的主题色 useEffect(() => { @@ -96,41 +100,65 @@ export default function App(): JSX.Element { onOpenWorkspace={openWorkspace} />
- - setBrowserOpen((open) => !open)} /> - +
+ setSettingsOpen(true)} - onPermissionMode={setPermissionMode} - prefillRef={prefillRef} + error={state.error} + retry={state.retry} + sessionError={state.sessionError} + pendingApprovals={state.approvals} + onDismissError={clearError} + onPickSuggestion={handlePickSuggestion} + onApplySelection={handleApplySelection} /> - } - /> + setSettingsOpen(true)} + onPermissionMode={setPermissionMode} + prefillRef={prefillRef} + /> + } + /> +
+ {browserOpen && ( + + )} +
void +}): JSX.Element { const name = title?.trim() const [maximized, setMaximized] = useState(false) @@ -15,7 +24,20 @@ export function TitleBar({ title }: { title?: string }): JSX.Element { return (
{name || '新会话'}
-
+
+ +
+
+ ))} + +
+ + {/* 工具条 */} +
+ + + +
+ + setAddress(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') submitAddress() + if (e.key === 'Escape') addressRef.current?.blur() + }} + placeholder="输入网址或搜索…" + spellCheck={false} + className="min-w-0 flex-1 bg-transparent text-[12px] text-zinc-700 outline-none placeholder:text-zinc-400" + /> +
+ +
+ + {/* 风险告知(首次使用门禁,Agent 与用户操作都会被拦截直到确认) */} + {riskBanner && ( +
+ +
+

+ Agent 将以你的身份操作网页。平台可能将自动化行为识别为异常,导致验证码、限流或账号风控;请勿在受管浏览器中登录支付等高敏感账号。 +

+ +
+
+ )} + + {/* 页面区域:原生 WebContentsView 对齐到这里的矩形 */} +
+ {showSlot && activeTab ? ( + + ) : ( +
+ + {riskBanner ? ( +

确认上方风险告知后即可打开网页

+ ) : ( + <> + +

+ 也可以直接让 Agent 帮你操作网页(browser_navigate / browser_observe 等工具) +

+ + )} +
+ )} +
+ + {/* Agent 操作轨迹 */} + {state?.trace && !importOpen && ( +
+ {state.trace} +
+ )} + + {/* 配置导入浮层(覆盖面板,出现时原生视图已隐藏) */} + {importOpen && ( + setImportOpen(false)} + onDetectSources={props.onDetectSources} + onImportProfile={props.onImportProfile} + /> + )} + + ) +} + +// ---- 配置导入浮层 ---- + +function ImportOverlay({ + state, + onClose, + onDetectSources, + onImportProfile +}: { + state: BrowserUiState | null + onClose: () => void + onDetectSources: () => Promise + onImportProfile: (sourceId: string, profileDir: string) => Promise +}): JSX.Element { + const [sources, setSources] = useState(null) + const [selected, setSelected] = useState<{ sourceId: string; profileDir: string } | null>(null) + const [busy, setBusy] = useState(false) + const [result, setResult] = useState(null) + const [error, setError] = useState(null) + + useEffect(() => { + onDetectSources() + .then((list) => { + setSources(list) + const first = list[0] + if (first?.profiles[0]) setSelected({ sourceId: first.id, profileDir: first.profiles[0].dir }) + }) + .catch((err) => setError(err instanceof Error ? err.message : String(err))) + }, [onDetectSources]) + + const runImport = (): void => { + if (!selected) return + setBusy(true) + setError(null) + setResult(null) + onImportProfile(selected.sourceId, selected.profileDir) + .then(setResult) + .catch((err) => setError(err instanceof Error ? err.message : String(err))) + .finally(() => setBusy(false)) + } + + return ( +
+
+
导入本机浏览器登录态
+ +
+ +
+

+ 选择本机浏览器的一个配置(Profile),把其中各网站的登录态(Cookie / Local Storage)导入到内置受管浏览器。 + 导入前请完全退出该浏览器,并关闭本面板的所有标签页。 +

+ + {state?.imported && ( +
+ 当前导入来源:{state.imported.browser} · {state.imported.profile} + {state.imported.cookies ? '(含 Cookie)' : ''} +
+ )} + + {sources === null && !error &&

正在检测本机浏览器…

} + {sources?.length === 0 &&

未检测到可导入的本机浏览器(支持 Chrome / Edge / Brave / Chromium)

} + +
+ {sources?.map((source) => ( +
+
+ + {source.name} + ({source.profiles.length} 个配置) +
+
+ {source.profiles.map((profile) => { + const active = selected?.sourceId === source.id && selected.profileDir === profile.dir + return ( + + ) + })} +
+
+ ))} +
+ + {error && ( +
{error}
+ )} + {result && ( +
+ {result.message} +
+ )} +
+ +
+ + +
+
+ ) +} diff --git a/apps/desktop/src/renderer/src/components/browser/BrowserSlot.tsx b/apps/desktop/src/renderer/src/components/browser/BrowserSlot.tsx new file mode 100644 index 0000000..9a24324 --- /dev/null +++ b/apps/desktop/src/renderer/src/components/browser/BrowserSlot.tsx @@ -0,0 +1,78 @@ +import * as React from 'react' + +/** + * 受管浏览器页面的 DOM 占位符。 + * + * 原生 WebContentsView 由主进程挂载在窗口 contentView 上,天然盖在所有 DOM 之上, + * CSS z-index 无法反转。这里量出占位矩形(ResizeObserver + 窗口 resize),经 IPC + * 同步给主进程做 setBounds,让原生视图精确对齐到面板布局。 + * + * revision 全局单调递增:快速切换 Tab 时旧 Slot 晚到的 IPC 不会覆盖新挂载的视图。 + */ + +let revisionCounter = 0 +function nextRevision(): number { + revisionCounter += 1 + return revisionCounter +} + +export function BrowserSlot({ tabId }: { tabId: string }): React.ReactElement { + const ref = React.useRef(null) + + React.useLayoutEffect(() => { + const element = ref.current + if (!element) return + let frame = 0 + + const commit = (visible: boolean): void => { + const rect = element.getBoundingClientRect() + void window.api.browserSetLayout({ + tabId, + visible: visible && rect.width > 4 && rect.height > 4, + revision: nextRevision(), + bounds: { + x: Math.round(rect.x), + y: Math.round(rect.y), + width: Math.round(rect.width), + height: Math.round(rect.height) + } + }) + } + const publish = (immediate = false): void => { + if (frame) cancelAnimationFrame(frame) + if (immediate) { + frame = 0 + commit(true) + return + } + frame = requestAnimationFrame(() => { + frame = 0 + commit(true) + }) + } + + const observer = new ResizeObserver(() => publish()) + observer.observe(element) + window.addEventListener('resize', onWindowResize) + // 挂载立即上报(不等一帧),避免快速切换时原生视图停留在隐藏状态 + publish(true) + + function onWindowResize(): void { + publish() + } + + return () => { + observer.disconnect() + window.removeEventListener('resize', onWindowResize) + if (frame) cancelAnimationFrame(frame) + void window.api.browserSetLayout({ + tabId, + visible: false, + revision: nextRevision(), + bounds: { x: 0, y: 0, width: 0, height: 0 } + }) + } + }, [tabId]) + + return
+} diff --git a/apps/desktop/src/renderer/src/hooks/useBrowser.ts b/apps/desktop/src/renderer/src/hooks/useBrowser.ts new file mode 100644 index 0000000..8769b80 --- /dev/null +++ b/apps/desktop/src/renderer/src/hooks/useBrowser.ts @@ -0,0 +1,57 @@ +import { useCallback, useEffect, useState } from 'react' +import type { BrowserImportResult, BrowserSource, BrowserUiState } from '@shared/types' + +/** + * 内置受管浏览器面板状态与操作。 + * 状态由主进程 BrowserController 广播(browser:event),这里只做订阅与动作封装。 + */ +export function useBrowser(): { + state: BrowserUiState | null + navigate: (input: string) => void + goBack: () => void + goForward: () => void + reload: () => void + newTab: () => void + selectTab: (tabId: string) => void + closeTab: (tabId: string) => void + closeBrowser: () => void + acknowledgeRisk: () => void + detectSources: () => Promise + importProfile: (sourceId: string, profileDir: string) => Promise +} { + const [state, setState] = useState(null) + + useEffect(() => { + void window.api.browserGetState().then(setState).catch(() => undefined) + return window.api.onBrowserEvent((event) => { + if (event.type === 'state') setState(event.state) + }) + }, []) + + const dispatchError = useCallback((err: unknown): void => { + console.error('[browser]', err instanceof Error ? err.message : String(err)) + }, []) + + const wrap = useCallback( + (fn: (...args: A[]) => Promise) => + (...args: A[]): void => { + void fn(...args).catch(dispatchError) + }, + [dispatchError] + ) + + return { + state, + navigate: wrap((input: string) => window.api.browserNavigate(input)), + goBack: wrap(() => window.api.browserGoBack()), + goForward: wrap(() => window.api.browserGoForward()), + reload: wrap(() => window.api.browserReload()), + newTab: wrap(() => window.api.browserNewTab()), + selectTab: wrap((tabId: string) => window.api.browserSelectTab(tabId)), + closeTab: wrap((tabId: string) => window.api.browserCloseTab(tabId)), + closeBrowser: wrap(() => window.api.browserClose()), + acknowledgeRisk: wrap(() => window.api.browserAcknowledgeRisk()), + detectSources: () => window.api.browserDetectSources(), + importProfile: (sourceId: string, profileDir: string) => window.api.browserImportProfile(sourceId, profileDir) + } +} diff --git a/apps/desktop/src/renderer/src/lib/api.ts b/apps/desktop/src/renderer/src/lib/api.ts index dfb3780..671140d 100644 --- a/apps/desktop/src/renderer/src/lib/api.ts +++ b/apps/desktop/src/renderer/src/lib/api.ts @@ -2,6 +2,10 @@ import type { AgentUiEvent, AppSettings, AttachmentInfo, + BrowserImportResult, + BrowserSource, + BrowserUiEvent, + BrowserUiState, ChatResources, CustomProviderConfig, ModelCatalog, @@ -16,7 +20,19 @@ import type { WorkspaceMeta } from '@shared/types' -export type { AgentUiEvent, AppSettings, ModelCatalog, ModelInfo, SessionMeta, SessionState, SessionStatsInfo } +export type { + AgentUiEvent, + AppSettings, + ModelCatalog, + ModelInfo, + SessionMeta, + SessionState, + SessionStatsInfo, + BrowserImportResult, + BrowserSource, + BrowserUiEvent, + BrowserUiState +} declare global { interface Window { @@ -59,6 +75,25 @@ declare global { setThemeColor: (id: string) => Promise resolveApproval: (id: string, approved: boolean) => Promise onEvent: (callback: (event: AgentUiEvent) => void) => () => void + browserGetState: () => Promise + browserNavigate: (input: string) => Promise + browserGoBack: () => Promise + browserGoForward: () => Promise + browserReload: () => Promise + browserNewTab: (url?: string) => Promise + browserSelectTab: (tabId: string) => Promise + browserCloseTab: (tabId?: string) => Promise + browserClose: () => Promise + browserAcknowledgeRisk: () => Promise + browserSetLayout: (layout: { + tabId: string + visible: boolean + revision: number + bounds: { x: number; y: number; width: number; height: number } + }) => Promise + browserDetectSources: () => Promise + browserImportProfile: (sourceId: string, profileDir: string) => Promise + onBrowserEvent: (callback: (event: BrowserUiEvent) => void) => () => void } } } diff --git a/apps/desktop/src/shared/types.ts b/apps/desktop/src/shared/types.ts index 213f542..835c23a 100644 --- a/apps/desktop/src/shared/types.ts +++ b/apps/desktop/src/shared/types.ts @@ -182,6 +182,66 @@ export interface SendImage { * 转发到渲染进程的 agent 事件。 * 已归一化为可结构化克隆的普通 JSON(message 原样透传,渲染层防御性读取)。 */ +// ===== 内置受管浏览器 ===== + +/** 浏览器标签页(主进程 → 渲染层面板) */ +export interface BrowserTabInfo { + tabId: string + url: string + title: string + loading: boolean +} + +/** 已导入的本地浏览器配置摘要(面板展示用) */ +export interface BrowserImportInfo { + browser: string + profile: string + at: string + cookies: boolean + localStorage: boolean + note?: string +} + +/** 浏览器面板完整状态(事件与查询共用同一结构) */ +export interface BrowserUiState { + /** 是否存在受管浏览器会话(有标签页即视为打开) */ + open: boolean + /** 风险告知是否已确认(首次使用门禁) */ + riskAcknowledged: boolean + tabs: BrowserTabInfo[] + activeTabId: string | null + /** 最近一次 Agent 浏览器操作描述(面板底部轨迹) */ + trace: string | null + imported?: BrowserImportInfo | null +} + +export type BrowserUiEvent = { type: 'state'; state: BrowserUiState } + +/** 本地浏览器(Chromium 系)下的一个 profile */ +export interface BrowserProfileMeta { + /** profile 目录名(Default / Profile 1 …) */ + dir: string + /** 展示名(Preferences 里的名称或目录名) */ + name: string +} + +/** 检测到的可导入浏览器来源 */ +export interface BrowserSource { + id: string + name: string + userDataDir: string + profiles: BrowserProfileMeta[] +} + +/** 导入结果(主进程返回;cookieNote 说明跳过原因) */ +export interface BrowserImportResult { + ok: boolean + cookies: boolean + localStorage: boolean + cookieNote?: string + message: string +} + export type AgentUiEvent = | { type: 'session_ready'; state: SessionState } | { type: 'session_error'; message: string }