import { session, WebContentsView, type BrowserWindow, type WebContents } from 'electron'; import type { BrowserStateEvent, ViewBounds, ViewMode, ViewportMessage } from './shared'; import { createProfilePopupHandler } from './profile-popup'; export interface ViewSpec { taskId: string; partition: string; allowedDomains: string[]; } /** 停放视图时留在窗内的像素数(见 park) */ const PARK_OVERLAP_PX = 2; interface ManagedView { spec: ViewSpec; view: WebContentsView; mode: ViewMode; targetId: string; } function hostMatches(host: string, pattern: string): boolean { const h = host.toLowerCase(); const p = pattern.toLowerCase().replace(/^https?:\/\//, ''); if (p.startsWith('*.')) { const suffix = p.slice(1); return h === p.slice(2) || h.endsWith(suffix); } return h === p; } /** 内嵌浏览器宿主:每个任务一个 WebContentsView,挂在主窗口里;browser-use 通过 CDP 接管它。 */ export class BrowserHost { private readonly views = new Map(); private shownTaskId: string | null = null; private bounds: ViewBounds | null = null; private visible = false; constructor( private readonly win: BrowserWindow, private readonly emit: (channel: string, payload: unknown) => void, ) { // 窗口变大时,停在右侧外面的页面可能露出来,重新摆一次 win.on('resize', () => this.layout()); } list(): Array<{ taskId: string; targetId: string; mode: ViewMode; url: string }> { return [...this.views.values()].map(item => ({ taskId: item.spec.taskId, targetId: item.targetId, mode: item.mode, url: item.view.webContents.getURL(), })); } async create(spec: ViewSpec): Promise<{ targetId: string }> { const existing = this.views.get(spec.taskId); if (existing) return { targetId: existing.targetId }; const partition = spec.partition.startsWith('persist:') ? spec.partition : `persist:${spec.partition}`; const viewSession = session.fromPartition(partition); viewSession.setPermissionRequestHandler((_contents, _permission, callback) => callback(false)); const view = new WebContentsView({ webPreferences: { session: viewSession, contextIsolation: true, nodeIntegration: false, sandbox: true, backgroundThrottling: false, devTools: false, }, }); const managed: ManagedView = { spec, view, mode: 'agent', targetId: '' }; this.views.set(spec.taskId, managed); this.win.contentView.addChildView(view); this.park(view); const page = view.webContents; this.wire(managed, page); await this.load(page, 'about:blank'); managed.targetId = await this.resolveTargetId(page); this.layout(); return { targetId: managed.targetId }; } dispose(taskId: string): void { const managed = this.views.get(taskId); if (!managed) return; this.views.delete(taskId); try { this.win.contentView.removeChildView(managed.view); } catch { /* window may be closing */ } try { managed.view.webContents.close(); } catch { /* already destroyed */ } if (this.shownTaskId === taskId) this.shownTaskId = null; this.layout(); } disposeAll(): void { for (const taskId of [...this.views.keys()]) this.dispose(taskId); } /** 渲染层告诉我们:现在展示哪个任务的页面、占位区域在哪里。 */ setViewport(message: ViewportMessage): void { this.visible = message.visible && Boolean(message.bounds); this.bounds = message.bounds ?? this.bounds; if (message.taskId) this.shownTaskId = message.taskId; this.layout(); // 渲染层刚打开面板(或刷新后重新选中任务)时补发一次页面状态,否则地址栏是空的 const shown = this.shownTaskId ? this.views.get(this.shownTaskId) : undefined; if (shown && this.visible) this.publish(shown); } setMode(taskId: string, mode: ViewMode): void { const managed = this.views.get(taskId); if (!managed) return; managed.mode = mode; this.publish(managed); } reveal(taskId: string): void { this.emit('browser:reveal', { taskId }); } hide(taskId: string): void { this.emit('browser:hide', { taskId }); } private layout(): void { for (const [taskId, managed] of this.views) { const show = this.visible && taskId === this.shownTaskId && this.bounds !== null; if (show && this.bounds) { managed.view.setBounds({ x: Math.max(0, Math.round(this.bounds.x)), y: Math.max(0, Math.round(this.bounds.y)), width: Math.max(1, Math.round(this.bounds.width)), height: Math.max(1, Math.round(this.bounds.height)), }); managed.view.setVisible(true); } else { this.park(managed.view); } } } /** * 不展示的页面停到窗口右边缘外,只留 2px 在窗内,而不是 setVisible(false)。 * 实测(Electron 33 / Win11 150% 缩放): * - setVisible(false) 或整个移出窗口 → Chromium 判定 occluded,页面 visibilityState=hidden,不再出帧, * CDP Page.captureScreenshot 一直挂着,browser-use 每步白等 15s 截图 + 15s 导航完成; * - 只要有 1px 落在窗内 → visible,截图 0.5s 内返回。留 2px 是为了抵消 DIP 取整误差。 */ private park(view: WebContentsView): void { const [winWidth] = this.win.getContentSize(); const size = this.bounds ?? { width: 1024, height: 720 }; view.setBounds({ x: Math.max(0, winWidth - PARK_OVERLAP_PX), y: 0, width: Math.max(1, Math.round(size.width)), height: Math.max(1, Math.round(size.height)), }); view.setVisible(true); } private allowed(url: string, spec: ViewSpec): boolean { if (url === 'about:blank' || url.startsWith('about:') || url.startsWith('data:') || url.startsWith('blob:')) return true; let host = ''; try { host = new URL(url).hostname; } catch { return false; } if (!host) return false; if (spec.allowedDomains.length === 0) return true; return spec.allowedDomains.some(pattern => hostMatches(host, pattern)); } private wire(managed: ManagedView, page: WebContents): void { const { spec } = managed; const popup = createProfilePopupHandler(page, { taskId: spec.taskId, isCurrent: () => this.views.get(spec.taskId) === managed, allowed: url => this.allowed(url, spec), diagnose: diagnostic => { console.info('[browser-host] popup', JSON.stringify(diagnostic)); }, }); page.on('input-event', (_event, input) => { popup.input(input.type); }); page.setWindowOpenHandler(details => popup.open(details)); page.on('will-navigate', (event, url) => { if (!this.allowed(url, spec)) { event.preventDefault(); console.warn(`[browser-host] 拦截导航(不在允许域名内): ${url}`); } }); page.on('will-redirect', (event, url) => { if (!this.allowed(url, spec)) event.preventDefault(); }); // Agent 模式下拦截用户键盘输入:提示用户先「接管」再操作,避免两边同时操作页面 page.on('before-input-event', (event, input) => { if (managed.mode !== 'agent') return; event.preventDefault(); if (input.type === 'keyDown') this.emit('browser:user-input', { taskId: spec.taskId }); }); const publish = (): void => this.publish(managed); page.on('did-start-loading', publish); page.on('did-stop-loading', publish); page.on('did-navigate', publish); page.on('did-navigate-in-page', publish); page.on('page-title-updated', publish); page.on('render-process-gone', (_event, details) => { console.warn(`[browser-host] 页面进程退出: ${details.reason}`); }); } private publish(managed: ManagedView): void { const page = managed.view.webContents; if (page.isDestroyed()) return; const event: BrowserStateEvent = { taskId: managed.spec.taskId, url: page.getURL(), title: page.getTitle(), loading: page.isLoading(), mode: managed.mode, }; this.emit('browser:state', event); } private load(page: WebContents, url: string): Promise { return new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error('页面初始化超时')), 15000); page.once('did-finish-load', () => { clearTimeout(timer); resolve(); }); page.loadURL(url).catch(error => { clearTimeout(timer); reject(error as Error); }); }); } /** 通过短暂附加调试器拿到该页面的 CDP targetId,供 browser-use 聚焦。 */ private async resolveTargetId(page: WebContents): Promise { const dbg = page.debugger; dbg.attach('1.3'); try { const result = await dbg.sendCommand('Target.getTargetInfo') as { targetInfo?: { targetId?: string } }; const targetId = result.targetInfo?.targetId; if (!targetId) throw new Error('无法获取页面 targetId'); return targetId; } finally { try { dbg.detach(); } catch { /* ignore */ } } } }