238 lines
8.9 KiB
TypeScript
238 lines
8.9 KiB
TypeScript
import { session, WebContentsView, type BrowserWindow, type WebContents } from 'electron';
|
||
import type { BrowserStateEvent, ViewBounds, ViewMode, ViewportMessage } from './shared';
|
||
import { createProfilePopupHandler } from './profile-popup';
|
||
|
||
export interface ViewSpec {
|
||
taskId: string;
|
||
partition: string;
|
||
allowedDomains: string[];
|
||
}
|
||
|
||
/** 停放视图时留在窗内的像素数（见 park） */
|
||
const PARK_OVERLAP_PX = 2;
|
||
|
||
interface ManagedView {
|
||
spec: ViewSpec;
|
||
view: WebContentsView;
|
||
mode: ViewMode;
|
||
targetId: string;
|
||
}
|
||
|
||
function hostMatches(host: string, pattern: string): boolean {
|
||
const h = host.toLowerCase();
|
||
const p = pattern.toLowerCase().replace(/^https?:\/\//, '');
|
||
if (p.startsWith('*.')) {
|
||
const suffix = p.slice(1);
|
||
return h === p.slice(2) || h.endsWith(suffix);
|
||
}
|
||
return h === p;
|
||
}
|
||
|
||
/** 内嵌浏览器宿主：每个任务一个 WebContentsView，挂在主窗口里；browser-use 通过 CDP 接管它。 */
|
||
export class BrowserHost {
|
||
private readonly views = new Map<string, ManagedView>();
|
||
private shownTaskId: string | null = null;
|
||
private bounds: ViewBounds | null = null;
|
||
private visible = false;
|
||
|
||
constructor(
|
||
private readonly win: BrowserWindow,
|
||
private readonly emit: (channel: string, payload: unknown) => void,
|
||
) {
|
||
// 窗口变大时，停在右侧外面的页面可能露出来，重新摆一次
|
||
win.on('resize', () => this.layout());
|
||
}
|
||
|
||
list(): Array<{ taskId: string; targetId: string; mode: ViewMode; url: string }> {
|
||
return [...this.views.values()].map(item => ({
|
||
taskId: item.spec.taskId, targetId: item.targetId, mode: item.mode, url: item.view.webContents.getURL(),
|
||
}));
|
||
}
|
||
|
||
async create(spec: ViewSpec): Promise<{ targetId: string }> {
|
||
const existing = this.views.get(spec.taskId);
|
||
if (existing) return { targetId: existing.targetId };
|
||
|
||
const partition = spec.partition.startsWith('persist:') ? spec.partition : `persist:${spec.partition}`;
|
||
const viewSession = session.fromPartition(partition);
|
||
viewSession.setPermissionRequestHandler((_contents, _permission, callback) => callback(false));
|
||
const view = new WebContentsView({
|
||
webPreferences: {
|
||
session: viewSession,
|
||
contextIsolation: true,
|
||
nodeIntegration: false,
|
||
sandbox: true,
|
||
backgroundThrottling: false,
|
||
devTools: false,
|
||
},
|
||
});
|
||
const managed: ManagedView = { spec, view, mode: 'agent', targetId: '' };
|
||
this.views.set(spec.taskId, managed);
|
||
this.win.contentView.addChildView(view);
|
||
this.park(view);
|
||
|
||
const page = view.webContents;
|
||
this.wire(managed, page);
|
||
await this.load(page, 'about:blank');
|
||
managed.targetId = await this.resolveTargetId(page);
|
||
this.layout();
|
||
return { targetId: managed.targetId };
|
||
}
|
||
|
||
dispose(taskId: string): void {
|
||
const managed = this.views.get(taskId);
|
||
if (!managed) return;
|
||
this.views.delete(taskId);
|
||
try { this.win.contentView.removeChildView(managed.view); } catch { /* window may be closing */ }
|
||
try { managed.view.webContents.close(); } catch { /* already destroyed */ }
|
||
if (this.shownTaskId === taskId) this.shownTaskId = null;
|
||
this.layout();
|
||
}
|
||
|
||
disposeAll(): void {
|
||
for (const taskId of [...this.views.keys()]) this.dispose(taskId);
|
||
}
|
||
|
||
/** 渲染层告诉我们：现在展示哪个任务的页面、占位区域在哪里。 */
|
||
setViewport(message: ViewportMessage): void {
|
||
this.visible = message.visible && Boolean(message.bounds);
|
||
this.bounds = message.bounds ?? this.bounds;
|
||
if (message.taskId) this.shownTaskId = message.taskId;
|
||
this.layout();
|
||
// 渲染层刚打开面板（或刷新后重新选中任务）时补发一次页面状态，否则地址栏是空的
|
||
const shown = this.shownTaskId ? this.views.get(this.shownTaskId) : undefined;
|
||
if (shown && this.visible) this.publish(shown);
|
||
}
|
||
|
||
setMode(taskId: string, mode: ViewMode): void {
|
||
const managed = this.views.get(taskId);
|
||
if (!managed) return;
|
||
managed.mode = mode;
|
||
this.publish(managed);
|
||
}
|
||
|
||
reveal(taskId: string): void {
|
||
this.emit('browser:reveal', { taskId });
|
||
}
|
||
|
||
hide(taskId: string): void {
|
||
this.emit('browser:hide', { taskId });
|
||
}
|
||
|
||
private layout(): void {
|
||
for (const [taskId, managed] of this.views) {
|
||
const show = this.visible && taskId === this.shownTaskId && this.bounds !== null;
|
||
if (show && this.bounds) {
|
||
managed.view.setBounds({
|
||
x: Math.max(0, Math.round(this.bounds.x)),
|
||
y: Math.max(0, Math.round(this.bounds.y)),
|
||
width: Math.max(1, Math.round(this.bounds.width)),
|
||
height: Math.max(1, Math.round(this.bounds.height)),
|
||
});
|
||
managed.view.setVisible(true);
|
||
} else {
|
||
this.park(managed.view);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 不展示的页面停到窗口右边缘外，只留 2px 在窗内，而不是 setVisible(false)。
|
||
* 实测（Electron 33 / Win11 150% 缩放）：
|
||
* - setVisible(false) 或整个移出窗口 → Chromium 判定 occluded，页面 visibilityState=hidden，不再出帧，
|
||
* CDP Page.captureScreenshot 一直挂着，browser-use 每步白等 15s 截图 + 15s 导航完成；
|
||
* - 只要有 1px 落在窗内 → visible，截图 0.5s 内返回。留 2px 是为了抵消 DIP 取整误差。
|
||
*/
|
||
private park(view: WebContentsView): void {
|
||
const [winWidth] = this.win.getContentSize();
|
||
const size = this.bounds ?? { width: 1024, height: 720 };
|
||
view.setBounds({
|
||
x: Math.max(0, winWidth - PARK_OVERLAP_PX),
|
||
y: 0,
|
||
width: Math.max(1, Math.round(size.width)),
|
||
height: Math.max(1, Math.round(size.height)),
|
||
});
|
||
view.setVisible(true);
|
||
}
|
||
|
||
private allowed(url: string, spec: ViewSpec): boolean {
|
||
if (url === 'about:blank' || url.startsWith('about:') || url.startsWith('data:') || url.startsWith('blob:')) return true;
|
||
let host = '';
|
||
try { host = new URL(url).hostname; } catch { return false; }
|
||
if (!host) return false;
|
||
if (spec.allowedDomains.length === 0) return true;
|
||
return spec.allowedDomains.some(pattern => hostMatches(host, pattern));
|
||
}
|
||
|
||
private wire(managed: ManagedView, page: WebContents): void {
|
||
const { spec } = managed;
|
||
const popup = createProfilePopupHandler(page, {
|
||
taskId: spec.taskId,
|
||
isCurrent: () => this.views.get(spec.taskId) === managed,
|
||
allowed: url => this.allowed(url, spec),
|
||
diagnose: diagnostic => { console.info('[browser-host] popup', JSON.stringify(diagnostic)); },
|
||
});
|
||
page.on('input-event', (_event, input) => { popup.input(input.type); });
|
||
page.setWindowOpenHandler(details => popup.open(details));
|
||
page.on('will-navigate', (event, url) => {
|
||
if (!this.allowed(url, spec)) {
|
||
event.preventDefault();
|
||
console.warn(`[browser-host] 拦截导航（不在允许域名内）: ${url}`);
|
||
}
|
||
});
|
||
page.on('will-redirect', (event, url) => {
|
||
if (!this.allowed(url, spec)) event.preventDefault();
|
||
});
|
||
// Agent 模式下拦截用户键盘输入：提示用户先「接管」再操作，避免两边同时操作页面
|
||
page.on('before-input-event', (event, input) => {
|
||
if (managed.mode !== 'agent') return;
|
||
event.preventDefault();
|
||
if (input.type === 'keyDown') this.emit('browser:user-input', { taskId: spec.taskId });
|
||
});
|
||
const publish = (): void => this.publish(managed);
|
||
page.on('did-start-loading', publish);
|
||
page.on('did-stop-loading', publish);
|
||
page.on('did-navigate', publish);
|
||
page.on('did-navigate-in-page', publish);
|
||
page.on('page-title-updated', publish);
|
||
page.on('render-process-gone', (_event, details) => {
|
||
console.warn(`[browser-host] 页面进程退出: ${details.reason}`);
|
||
});
|
||
}
|
||
|
||
private publish(managed: ManagedView): void {
|
||
const page = managed.view.webContents;
|
||
if (page.isDestroyed()) return;
|
||
const event: BrowserStateEvent = {
|
||
taskId: managed.spec.taskId,
|
||
url: page.getURL(),
|
||
title: page.getTitle(),
|
||
loading: page.isLoading(),
|
||
mode: managed.mode,
|
||
};
|
||
this.emit('browser:state', event);
|
||
}
|
||
|
||
private load(page: WebContents, url: string): Promise<void> {
|
||
return new Promise((resolve, reject) => {
|
||
const timer = setTimeout(() => reject(new Error('页面初始化超时')), 15000);
|
||
page.once('did-finish-load', () => { clearTimeout(timer); resolve(); });
|
||
page.loadURL(url).catch(error => { clearTimeout(timer); reject(error as Error); });
|
||
});
|
||
}
|
||
|
||
/** 通过短暂附加调试器拿到该页面的 CDP targetId，供 browser-use 聚焦。 */
|
||
private async resolveTargetId(page: WebContents): Promise<string> {
|
||
const dbg = page.debugger;
|
||
dbg.attach('1.3');
|
||
try {
|
||
const result = await dbg.sendCommand('Target.getTargetInfo') as { targetInfo?: { targetId?: string } };
|
||
const targetId = result.targetInfo?.targetId;
|
||
if (!targetId) throw new Error('无法获取页面 targetId');
|
||
return targetId;
|
||
} finally {
|
||
try { dbg.detach(); } catch { /* ignore */ }
|
||
}
|
||
}
|
||
}
|