235 lines
15 KiB
JavaScript
235 lines
15 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { spawn } from 'node:child_process';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { pythonPath, root } from './server.mjs';
|
|
import { chromium } from 'playwright';
|
|
import { createServer } from 'node:net';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { stopOwnedChild } from './processes.mjs';
|
|
import { acceptLearningTools, acceptLearningToolsUI, startLearningFixture } from './learning-tools-acceptance.mjs';
|
|
|
|
// This acceptance run only uses isolated local fixture pages and its own processes.
|
|
const run = path.join(root, '.tmp', `learning-acceptance-${Date.now()}`);
|
|
fs.mkdirSync(run, { recursive: true });
|
|
async function unusedPort() {
|
|
const listener = createServer();
|
|
await new Promise((resolve, reject) => { listener.once('error', reject); listener.listen(0, '127.0.0.1', resolve); });
|
|
const port = listener.address().port;
|
|
await new Promise(resolve => listener.close(resolve));
|
|
return port;
|
|
}
|
|
const apiPort = Number(process.env.ASSISTANT_TEST_API_PORT || await unusedPort());
|
|
let uiPort = Number(process.env.ASSISTANT_TEST_UI_PORT || await unusedPort());
|
|
if (uiPort === apiPort) uiPort = await unusedPort();
|
|
const apiBase = `http://127.0.0.1:${apiPort}`;
|
|
const uiBase = `http://127.0.0.1:${uiPort}`;
|
|
const environment = {
|
|
...process.env,
|
|
// Never inherit the actual development database loaded from .env.
|
|
ASSISTANT_DATABASE_URL: process.env.ASSISTANT_TEST_DATABASE_URL || '',
|
|
ASSISTANT_DATABASE_SCHEMA: 'assistant_learning_e2e_' + randomUUID().replaceAll('-', ''),
|
|
PYTHONUTF8: '1',
|
|
ASSISTANT_NODE: process.execPath,
|
|
ASSISTANT_LEARNING_HOME: path.join(run, 'learning'),
|
|
ASSISTANT_ALLOWED_ORIGINS: uiBase,
|
|
ASSISTANT_BROWSER_HEADLESS: '1',
|
|
ASSISTANT_BROWSER_CHANNEL: process.env.ASSISTANT_TEST_BROWSER_CHANNEL || (process.platform === 'win32' ? 'msedge' : 'chromium'),
|
|
ASSISTANT_API_TARGET: apiBase,
|
|
ASSISTANT_INSTANCE_ID: randomUUID(),
|
|
};
|
|
const children = new Set();
|
|
const errors = [];
|
|
let browser;
|
|
let page;
|
|
let fixture;
|
|
function launch(command, args, name) {
|
|
const child = spawn(command, args, { cwd: root, env: environment, windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] });
|
|
child.assistantGraceful = name.startsWith('server');
|
|
children.add(child);
|
|
const log = fs.createWriteStream(path.join(run, name + '.log'));
|
|
child.stdout.pipe(log); child.stderr.pipe(log);
|
|
child.on('error', error => errors.push(name + ': ' + error.message));
|
|
return child;
|
|
}
|
|
async function stop(child) {
|
|
await stopOwnedChild(child, { graceful: child?.assistantGraceful, timeout: child?.assistantGraceful ? 20000 : 3000 });
|
|
children.delete(child);
|
|
}
|
|
async function ready(url, child) {
|
|
for (let i = 0; i < 100; i++) {
|
|
if (child.exitCode !== null) throw Error('Service exited: ' + url);
|
|
try {
|
|
const response = await fetch(url, { signal: AbortSignal.timeout(1500) });
|
|
if (response.ok && (await response.json()).instance_id === environment.ASSISTANT_INSTANCE_ID) return;
|
|
} catch {}
|
|
await delay(250);
|
|
}
|
|
throw Error('Service not ready: ' + url);
|
|
}
|
|
class Client {
|
|
cookies = new Map();
|
|
csrf = '';
|
|
async request(method, route, body, expected = 200) {
|
|
const response = await fetch(apiBase + route, {
|
|
method,
|
|
headers: { Cookie: [...this.cookies].map(([key, value]) => `${key}=${value}`).join('; '), Origin: apiBase,
|
|
...(body === undefined ? {} : { 'Content-Type': 'application/json', 'X-CSRF-Token': this.csrf }) },
|
|
body: body === undefined ? undefined : JSON.stringify(body),
|
|
signal: AbortSignal.timeout(120000),
|
|
});
|
|
for (const cookie of response.headers.getSetCookie()) {
|
|
const pair = cookie.split(';', 1)[0], split = pair.indexOf('=');
|
|
this.cookies.set(pair.slice(0, split), pair.slice(split + 1));
|
|
}
|
|
const contentType = response.headers.get('content-type') || '';
|
|
const data = contentType.includes('application/json') ? await response.json() : Buffer.from(await response.arrayBuffer());
|
|
assert.equal(response.status, expected, `${method} ${route}: ${JSON.stringify(data)}`);
|
|
if (data?.csrf_token) this.csrf = data.csrf_token;
|
|
return data;
|
|
}
|
|
}
|
|
const username = 'admin';
|
|
const password = '123456';
|
|
try {
|
|
const backendArgs = ['-m', 'scripts.serve', '--port', String(apiPort)];
|
|
let backend = launch(pythonPath(), backendArgs, 'server');
|
|
await ready(apiBase + '/api/health', backend);
|
|
const client = new Client();
|
|
const initial = await client.request('GET', '/api/auth/status');
|
|
assert.equal(initial.initialized, true);
|
|
assert.equal(initial.authenticated, false);
|
|
await client.request('GET', '/api/learning/tasks', undefined, 401);
|
|
const setup = await client.request('POST', '/api/auth/login', { username, password });
|
|
assert.equal(setup.authenticated, true);
|
|
await client.request('POST', '/api/auth/setup', { username: 'other-maintainer', password: 'OnlySetupFixture-123' }, 409);
|
|
const empty = await client.request('GET', '/api/learning/tasks');
|
|
assert.equal(empty.items.length, 0);
|
|
const demo = await client.request('POST', '/api/learning/demo', {});
|
|
assert.ok(demo.task_id);
|
|
const task = await client.request('GET', '/api/learning/tasks/' + demo.task_id);
|
|
assert.ok(task.observations.length >= 3, 'Demo must save actual observations');
|
|
assert.ok(task.steps.length >= 4, 'Demo must record search, detail and read steps');
|
|
assert.ok(task.steps.some(step => step.kind === 'read' && step.output_name === 'comments'));
|
|
assert.equal(task.candidates.length, 1);
|
|
const candidate = task.candidates[0];
|
|
const digest = candidate.digest;
|
|
const browserRuns = task.validations.filter(record => record.kind === 'browser');
|
|
assert.equal(browserRuns.length, 2);
|
|
assert.ok(browserRuns.every(record => record.status === 'passed'));
|
|
assert.equal(new Set(browserRuns.map(record => JSON.stringify(record.parameters))).size, 2);
|
|
assert.ok(browserRuns.every(record => JSON.stringify(record.actual_outputs).includes(record.parameters.keyword)));
|
|
assert.ok(browserRuns.every(record => !JSON.stringify(record.environment).includes(process.platform === 'win32' ? 'macos' : 'windows')));
|
|
assert.ok(task.observations.some(record => record.screenshot_url));
|
|
const screenshot = task.observations.find(record => record.screenshot_url).screenshot_url;
|
|
const image = await client.request('GET', screenshot);
|
|
assert.equal(image.subarray(1, 4).toString(), 'PNG');
|
|
const precheck = await client.request('POST', '/api/learning/candidates/' + candidate.id + '/publish-check', {});
|
|
assert.equal(precheck.ready, false, 'Single OS acceptance must not pretend to cover both OSes');
|
|
assert.ok(precheck.checks.some(check => !check.passed));
|
|
// Demo deliberately closes its own browser. A new actual run must use an
|
|
// explicitly opened and confirmed browser context again.
|
|
await client.request('POST', '/api/learning/tasks/' + demo.task_id + '/browser/open', {});
|
|
await client.request('POST', '/api/learning/tasks/' + demo.task_id + '/browser/ready', { identity_label: '本机示例维护者' });
|
|
await client.request('POST', '/api/learning/candidates/' + candidate.id + '/verify', {
|
|
parameters: { keyword: '建材' }, expected_outputs: { comments: '肯定不存在的预期内容' }, sample_name: '故意不匹配的预期',
|
|
});
|
|
const afterFailure = await client.request('GET', '/api/learning/tasks/' + demo.task_id);
|
|
assert.equal(afterFailure.candidates[0].digest, digest);
|
|
assert.ok(afterFailure.validations.some(record => record.kind === 'browser' && record.status === 'failed'));
|
|
const count = afterFailure.validations.length;
|
|
fixture = await startLearningFixture();
|
|
const tools = await acceptLearningTools({ client, apiBase, fixtureBase: fixture.base, username, password });
|
|
assert.equal(fixture.receivedWrites(), 0, 'The read-only fixture must not receive blocked POSTs');
|
|
const validationTotal = (await client.request('GET', '/api/learning/validations')).items.length;
|
|
await stop(backend);
|
|
assert.equal(backend.exitCode, 0, 'The owned backend must finish its normal lifespan shutdown');
|
|
backend = launch(pythonPath(), backendArgs, 'server-restart');
|
|
await ready(apiBase + '/api/health', backend);
|
|
const restored = await client.request('GET', '/api/learning/tasks/' + demo.task_id);
|
|
assert.equal(restored.validations.length, count);
|
|
assert.equal(restored.candidates[0].digest, digest);
|
|
const restoredTools = await client.request('GET', '/api/learning/candidates/' + tools.candidate_id);
|
|
assert.equal(restoredTools.digest, tools.digest);
|
|
assert.equal(restoredTools.quality.state, 'review_required');
|
|
assert.deepEqual(restoredTools.validations.find(item => item.id === tools.validation_id).quality_observations.map(item => item.state), ['waiting', 'alert']);
|
|
const restoredFailure = await client.request('GET', '/api/learning/tasks/' + tools.failure_task_id);
|
|
assert.equal(restoredFailure.steps.length, 0);
|
|
assert.equal(restoredFailure.observations.filter(item => item.source === 'action_failure').length, tools.failure_evidence);
|
|
|
|
if (process.argv.includes('--api-only')) {
|
|
fs.writeFileSync(path.join(run, 'summary.json'), JSON.stringify({ task_id: task.id, tools, persisted_validations: validationTotal }, null, 2));
|
|
console.log(JSON.stringify({ result: 'passed', output: run, task_id: task.id, browser_validations: browserRuns.length, persisted_validations: validationTotal, tools }));
|
|
process.exitCode = 0;
|
|
} else {
|
|
|
|
const frontend = launch(process.execPath, [path.join(root, 'apps/admin/node_modules/vite/bin/vite.js'), '--config', path.join(root, 'apps/admin/vite.config.ts'), '--host', '127.0.0.1', '--port', String(uiPort), '--strictPort'], 'admin');
|
|
await ready(uiBase + '/api/health', frontend);
|
|
browser = await chromium.launch({ headless: true, channel: environment.ASSISTANT_BROWSER_CHANNEL });
|
|
page = await browser.newPage({ viewport: { width: 1480, height: 1050 } });
|
|
page.on('pageerror', error => errors.push(error.message));
|
|
await page.goto(uiBase);
|
|
await page.getByLabel(/账号|用户名/).first().fill(username);
|
|
await page.getByLabel('密码', { exact: true }).fill(password);
|
|
await page.getByTestId('auth-submit').click();
|
|
await page.getByRole('heading', { name: '学习批次', exact: true }).waitFor();
|
|
await page.getByTestId('task-list').waitFor();
|
|
await page.screenshot({ path: path.join(run, 'tasks.png'), fullPage: true, animations: 'disabled' });
|
|
await page.getByRole('button', { name: '账本总览', exact: true }).click();
|
|
await page.getByRole('heading', { name: '账本总览', exact: true }).waitFor();
|
|
await page.screenshot({ path: path.join(run, 'overview.png'), fullPage: true, animations: 'disabled' });
|
|
await page.getByRole('button', { name: '能力实体', exact: true }).click();
|
|
await page.getByRole('textbox', { name: '搜索能力', exact: true }).fill(candidate.id);
|
|
await page.getByRole('button', { name: /查看.*详情/ }).first().click();
|
|
await page.getByRole('tab', { name: '使用参数', exact: true }).waitFor();
|
|
await page.getByRole('tab', { name: `测试记录 ${count}`, exact: true }).waitFor();
|
|
await page.screenshot({ path: path.join(run, 'capability.png'), fullPage: true, animations: 'disabled' });
|
|
await page.getByRole('tab', { name: '操作链', exact: true }).click();
|
|
assert.equal(await page.locator('table tbody tr').count(), task.steps.length);
|
|
await page.getByRole('button', { name: '发布预检', exact: true }).click();
|
|
await page.getByText('仍有条件需要补齐', { exact: true }).waitFor();
|
|
await page.getByText('macOS 实际试跑', { exact: true }).waitFor();
|
|
await page.screenshot({ path: path.join(run, 'preflight.png'), fullPage: true, animations: 'disabled' });
|
|
await page.getByRole('button', { name: /关闭|close/i }).last().click();
|
|
await page.getByRole('button', { name: '脚本测试', exact: true }).click();
|
|
await page.getByRole('heading', { name: '脚本测试', exact: true }).waitFor();
|
|
await page.getByTestId('validation-detail').first().waitFor();
|
|
assert.equal(await page.getByTestId('validation-detail').count(), validationTotal);
|
|
await page.screenshot({ path: path.join(run, 'validations.png'), fullPage: true, animations: 'disabled' });
|
|
await page.getByTestId('validation-detail').first().click();
|
|
await page.getByTestId('validation-record').waitFor();
|
|
await page.screenshot({ path: path.join(run, 'validation-detail.png'), fullPage: true, animations: 'disabled' });
|
|
await page.getByRole('button', { name: /关闭|close/i }).last().click();
|
|
const toolsUI = await acceptLearningToolsUI({ page, client, uiBase, tools });
|
|
await page.screenshot({ path: path.join(run, 'bounded-tools-ui.png'), fullPage: true, animations: 'disabled' });
|
|
await page.getByTestId('nav-tasks').click();
|
|
await page.getByRole('button', { name: '新建批次', exact: true }).click();
|
|
const dialog = page.getByRole('dialog');
|
|
assert.equal(await dialog.getByLabel('页面入口', { exact: true }).inputValue(), 'https://www.xiaohongshu.com/explore');
|
|
await dialog.getByLabel('本地账号引用', { exact: true }).fill('xhs-fixture');
|
|
await dialog.getByLabel('任务参数', { exact: true }).fill(JSON.stringify({ keyword: '建材' }));
|
|
await dialog.getByRole('button', { name: '创建任务', exact: true }).click();
|
|
await page.getByRole('heading', { name: '小红书搜索与评论读取', exact: true }).waitFor();
|
|
await page.getByRole('button', { name: /打开浏览器/ }).waitFor();
|
|
await page.screenshot({ path: path.join(run, 'new-xiaohongshu-task.png'), fullPage: true, animations: 'disabled' });
|
|
const createdTasks = await client.request('GET', '/api/learning/tasks');
|
|
assert.ok(createdTasks.items.some(item => item.platform === 'xiaohongshu' && item.account_key === 'xhs-fixture' && item.browser_status === 'closed'));
|
|
await browser.close(); browser = null;
|
|
assert.deepEqual(errors, []);
|
|
const finalValidationTotal = (await client.request('GET', '/api/learning/validations')).items.length;
|
|
fs.writeFileSync(path.join(run, 'summary.json'), JSON.stringify({ task_id: task.id, candidate_id: candidate.id, digest,
|
|
observations: task.observations.length, steps: task.steps.length, browser_validations: browserRuns.length,
|
|
persisted_validations: finalValidationTotal, tools, toolsUI, publish_ready: precheck.ready, errors }, null, 2));
|
|
console.log(JSON.stringify({ result: 'passed', output: run, task_id: task.id, browser_validations: browserRuns.length, persisted_validations: finalValidationTotal, tools, toolsUI }));
|
|
}
|
|
} catch (error) {
|
|
if (page && !page.isClosed()) await page.screenshot({ path: path.join(run, 'failure.png'), fullPage: true }).catch(() => {});
|
|
console.error('Acceptance artifacts: ' + run);
|
|
throw error;
|
|
} finally {
|
|
if (browser) await browser.close().catch(() => {});
|
|
for (const child of [...children]) await stop(child);
|
|
if (fixture) await fixture.close();
|
|
}
|