austin
/
assistant
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
assistant/scripts/test-learning.mjs

235 lines
15 KiB
JavaScript
Raw Permalink Blame History

import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { spawn } from 'node:child_process';
import { setTimeout as delay } from 'node:timers/promises';
import { pythonPath, root } from './server.mjs';
import { chromium } from 'playwright';
import { createServer } from 'node:net';
import { randomUUID } from 'node:crypto';
import { stopOwnedChild } from './processes.mjs';
import { acceptLearningTools, acceptLearningToolsUI, startLearningFixture } from './learning-tools-acceptance.mjs';
// This acceptance run only uses isolated local fixture pages and its own processes.
const run = path.join(root, '.tmp', `learning-acceptance-${Date.now()}`);
fs.mkdirSync(run, { recursive: true });
async function unusedPort() {
const listener = createServer();
await new Promise((resolve, reject) => { listener.once('error', reject); listener.listen(0, '127.0.0.1', resolve); });
const port = listener.address().port;
await new Promise(resolve => listener.close(resolve));
return port;
}
const apiPort = Number(process.env.ASSISTANT_TEST_API_PORT || await unusedPort());
let uiPort = Number(process.env.ASSISTANT_TEST_UI_PORT || await unusedPort());
if (uiPort === apiPort) uiPort = await unusedPort();
const apiBase = `http://127.0.0.1:${apiPort}`;
const uiBase = `http://127.0.0.1:${uiPort}`;
const environment = {
...process.env,
// Never inherit the actual development database loaded from .env.
ASSISTANT_DATABASE_URL: process.env.ASSISTANT_TEST_DATABASE_URL || '',
ASSISTANT_DATABASE_SCHEMA: 'assistant_learning_e2e_' + randomUUID().replaceAll('-', ''),
PYTHONUTF8: '1',
ASSISTANT_NODE: process.execPath,
ASSISTANT_LEARNING_HOME: path.join(run, 'learning'),
ASSISTANT_ALLOWED_ORIGINS: uiBase,
ASSISTANT_BROWSER_HEADLESS: '1',
ASSISTANT_BROWSER_CHANNEL: process.env.ASSISTANT_TEST_BROWSER_CHANNEL || (process.platform === 'win32' ? 'msedge' : 'chromium'),
ASSISTANT_API_TARGET: apiBase,
ASSISTANT_INSTANCE_ID: randomUUID(),
};
const children = new Set();
const errors = [];
let browser;
let page;
let fixture;
function launch(command, args, name) {
const child = spawn(command, args, { cwd: root, env: environment, windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] });
child.assistantGraceful = name.startsWith('server');
children.add(child);
const log = fs.createWriteStream(path.join(run, name + '.log'));
child.stdout.pipe(log); child.stderr.pipe(log);
child.on('error', error => errors.push(name + ': ' + error.message));
return child;
}
async function stop(child) {
await stopOwnedChild(child, { graceful: child?.assistantGraceful, timeout: child?.assistantGraceful ? 20000 : 3000 });
children.delete(child);
}
async function ready(url, child) {
for (let i = 0; i < 100; i++) {
if (child.exitCode !== null) throw Error('Service exited: ' + url);
try {
const response = await fetch(url, { signal: AbortSignal.timeout(1500) });
if (response.ok && (await response.json()).instance_id === environment.ASSISTANT_INSTANCE_ID) return;
} catch {}
await delay(250);
}
throw Error('Service not ready: ' + url);
}
class Client {
cookies = new Map();
csrf = '';
async request(method, route, body, expected = 200) {
const response = await fetch(apiBase + route, {
method,
headers: { Cookie: [...this.cookies].map(([key, value]) => `${key}=${value}`).join('; '), Origin: apiBase,
...(body === undefined ? {} : { 'Content-Type': 'application/json', 'X-CSRF-Token': this.csrf }) },
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(120000),
});
for (const cookie of response.headers.getSetCookie()) {
const pair = cookie.split(';', 1)[0], split = pair.indexOf('=');
this.cookies.set(pair.slice(0, split), pair.slice(split + 1));
}
const contentType = response.headers.get('content-type') || '';
const data = contentType.includes('application/json') ? await response.json() : Buffer.from(await response.arrayBuffer());
assert.equal(response.status, expected, `${method} ${route}: ${JSON.stringify(data)}`);
if (data?.csrf_token) this.csrf = data.csrf_token;
return data;
}
}
const username = 'admin';
const password = '123456';
try {
const backendArgs = ['-m', 'scripts.serve', '--port', String(apiPort)];
let backend = launch(pythonPath(), backendArgs, 'server');
await ready(apiBase + '/api/health', backend);
const client = new Client();
const initial = await client.request('GET', '/api/auth/status');
assert.equal(initial.initialized, true);
assert.equal(initial.authenticated, false);
await client.request('GET', '/api/learning/tasks', undefined, 401);
const setup = await client.request('POST', '/api/auth/login', { username, password });
assert.equal(setup.authenticated, true);
await client.request('POST', '/api/auth/setup', { username: 'other-maintainer', password: 'OnlySetupFixture-123' }, 409);
const empty = await client.request('GET', '/api/learning/tasks');
assert.equal(empty.items.length, 0);
const demo = await client.request('POST', '/api/learning/demo', {});
assert.ok(demo.task_id);
const task = await client.request('GET', '/api/learning/tasks/' + demo.task_id);
assert.ok(task.observations.length >= 3, 'Demo must save actual observations');
assert.ok(task.steps.length >= 4, 'Demo must record search, detail and read steps');
assert.ok(task.steps.some(step => step.kind === 'read' && step.output_name === 'comments'));
assert.equal(task.candidates.length, 1);
const candidate = task.candidates[0];
const digest = candidate.digest;
const browserRuns = task.validations.filter(record => record.kind === 'browser');
assert.equal(browserRuns.length, 2);
assert.ok(browserRuns.every(record => record.status === 'passed'));
assert.equal(new Set(browserRuns.map(record => JSON.stringify(record.parameters))).size, 2);
assert.ok(browserRuns.every(record => JSON.stringify(record.actual_outputs).includes(record.parameters.keyword)));
assert.ok(browserRuns.every(record => !JSON.stringify(record.environment).includes(process.platform === 'win32' ? 'macos' : 'windows')));
assert.ok(task.observations.some(record => record.screenshot_url));
const screenshot = task.observations.find(record => record.screenshot_url).screenshot_url;
const image = await client.request('GET', screenshot);
assert.equal(image.subarray(1, 4).toString(), 'PNG');
const precheck = await client.request('POST', '/api/learning/candidates/' + candidate.id + '/publish-check', {});
assert.equal(precheck.ready, false, 'Single OS acceptance must not pretend to cover both OSes');
assert.ok(precheck.checks.some(check => !check.passed));
// Demo deliberately closes its own browser. A new actual run must use an
// explicitly opened and confirmed browser context again.
await client.request('POST', '/api/learning/tasks/' + demo.task_id + '/browser/open', {});
await client.request('POST', '/api/learning/tasks/' + demo.task_id + '/browser/ready', { identity_label: '本机示例维护者' });
await client.request('POST', '/api/learning/candidates/' + candidate.id + '/verify', {
parameters: { keyword: '建材' }, expected_outputs: { comments: '肯定不存在的预期内容' }, sample_name: '故意不匹配的预期',
});
const afterFailure = await client.request('GET', '/api/learning/tasks/' + demo.task_id);
assert.equal(afterFailure.candidates[0].digest, digest);
assert.ok(afterFailure.validations.some(record => record.kind === 'browser' && record.status === 'failed'));
const count = afterFailure.validations.length;
fixture = await startLearningFixture();
const tools = await acceptLearningTools({ client, apiBase, fixtureBase: fixture.base, username, password });
assert.equal(fixture.receivedWrites(), 0, 'The read-only fixture must not receive blocked POSTs');
const validationTotal = (await client.request('GET', '/api/learning/validations')).items.length;
await stop(backend);
assert.equal(backend.exitCode, 0, 'The owned backend must finish its normal lifespan shutdown');
backend = launch(pythonPath(), backendArgs, 'server-restart');
await ready(apiBase + '/api/health', backend);
const restored = await client.request('GET', '/api/learning/tasks/' + demo.task_id);
assert.equal(restored.validations.length, count);
assert.equal(restored.candidates[0].digest, digest);
const restoredTools = await client.request('GET', '/api/learning/candidates/' + tools.candidate_id);
assert.equal(restoredTools.digest, tools.digest);
assert.equal(restoredTools.quality.state, 'review_required');
assert.deepEqual(restoredTools.validations.find(item => item.id === tools.validation_id).quality_observations.map(item => item.state), ['waiting', 'alert']);
const restoredFailure = await client.request('GET', '/api/learning/tasks/' + tools.failure_task_id);
assert.equal(restoredFailure.steps.length, 0);
assert.equal(restoredFailure.observations.filter(item => item.source === 'action_failure').length, tools.failure_evidence);
if (process.argv.includes('--api-only')) {
fs.writeFileSync(path.join(run, 'summary.json'), JSON.stringify({ task_id: task.id, tools, persisted_validations: validationTotal }, null, 2));
console.log(JSON.stringify({ result: 'passed', output: run, task_id: task.id, browser_validations: browserRuns.length, persisted_validations: validationTotal, tools }));
process.exitCode = 0;
} else {
const frontend = launch(process.execPath, [path.join(root, 'apps/admin/node_modules/vite/bin/vite.js'), '--config', path.join(root, 'apps/admin/vite.config.ts'), '--host', '127.0.0.1', '--port', String(uiPort), '--strictPort'], 'admin');
await ready(uiBase + '/api/health', frontend);
browser = await chromium.launch({ headless: true, channel: environment.ASSISTANT_BROWSER_CHANNEL });
page = await browser.newPage({ viewport: { width: 1480, height: 1050 } });
page.on('pageerror', error => errors.push(error.message));
await page.goto(uiBase);
await page.getByLabel(/账号|用户名/).first().fill(username);
await page.getByLabel('密码', { exact: true }).fill(password);
await page.getByTestId('auth-submit').click();
await page.getByRole('heading', { name: '学习批次', exact: true }).waitFor();
await page.getByTestId('task-list').waitFor();
await page.screenshot({ path: path.join(run, 'tasks.png'), fullPage: true, animations: 'disabled' });
await page.getByRole('button', { name: '账本总览', exact: true }).click();
await page.getByRole('heading', { name: '账本总览', exact: true }).waitFor();
await page.screenshot({ path: path.join(run, 'overview.png'), fullPage: true, animations: 'disabled' });
await page.getByRole('button', { name: '能力实体', exact: true }).click();
await page.getByRole('textbox', { name: '搜索能力', exact: true }).fill(candidate.id);
await page.getByRole('button', { name: /查看.*详情/ }).first().click();
await page.getByRole('tab', { name: '使用参数', exact: true }).waitFor();
await page.getByRole('tab', { name: `测试记录 ${count}`, exact: true }).waitFor();
await page.screenshot({ path: path.join(run, 'capability.png'), fullPage: true, animations: 'disabled' });
await page.getByRole('tab', { name: '操作链', exact: true }).click();
assert.equal(await page.locator('table tbody tr').count(), task.steps.length);
await page.getByRole('button', { name: '发布预检', exact: true }).click();
await page.getByText('仍有条件需要补齐', { exact: true }).waitFor();
await page.getByText('macOS 实际试跑', { exact: true }).waitFor();
await page.screenshot({ path: path.join(run, 'preflight.png'), fullPage: true, animations: 'disabled' });
await page.getByRole('button', { name: /关闭|close/i }).last().click();
await page.getByRole('button', { name: '脚本测试', exact: true }).click();
await page.getByRole('heading', { name: '脚本测试', exact: true }).waitFor();
await page.getByTestId('validation-detail').first().waitFor();
assert.equal(await page.getByTestId('validation-detail').count(), validationTotal);
await page.screenshot({ path: path.join(run, 'validations.png'), fullPage: true, animations: 'disabled' });
await page.getByTestId('validation-detail').first().click();
await page.getByTestId('validation-record').waitFor();
await page.screenshot({ path: path.join(run, 'validation-detail.png'), fullPage: true, animations: 'disabled' });
await page.getByRole('button', { name: /关闭|close/i }).last().click();
const toolsUI = await acceptLearningToolsUI({ page, client, uiBase, tools });
await page.screenshot({ path: path.join(run, 'bounded-tools-ui.png'), fullPage: true, animations: 'disabled' });
await page.getByTestId('nav-tasks').click();
await page.getByRole('button', { name: '新建批次', exact: true }).click();
const dialog = page.getByRole('dialog');
assert.equal(await dialog.getByLabel('页面入口', { exact: true }).inputValue(), 'https://www.xiaohongshu.com/explore');
await dialog.getByLabel('本地账号引用', { exact: true }).fill('xhs-fixture');
await dialog.getByLabel('任务参数', { exact: true }).fill(JSON.stringify({ keyword: '建材' }));
await dialog.getByRole('button', { name: '创建任务', exact: true }).click();
await page.getByRole('heading', { name: '小红书搜索与评论读取', exact: true }).waitFor();
await page.getByRole('button', { name: /打开浏览器/ }).waitFor();
await page.screenshot({ path: path.join(run, 'new-xiaohongshu-task.png'), fullPage: true, animations: 'disabled' });
const createdTasks = await client.request('GET', '/api/learning/tasks');
assert.ok(createdTasks.items.some(item => item.platform === 'xiaohongshu' && item.account_key === 'xhs-fixture' && item.browser_status === 'closed'));
await browser.close(); browser = null;
assert.deepEqual(errors, []);
const finalValidationTotal = (await client.request('GET', '/api/learning/validations')).items.length;
fs.writeFileSync(path.join(run, 'summary.json'), JSON.stringify({ task_id: task.id, candidate_id: candidate.id, digest,
observations: task.observations.length, steps: task.steps.length, browser_validations: browserRuns.length,
persisted_validations: finalValidationTotal, tools, toolsUI, publish_ready: precheck.ready, errors }, null, 2));
console.log(JSON.stringify({ result: 'passed', output: run, task_id: task.id, browser_validations: browserRuns.length, persisted_validations: finalValidationTotal, tools, toolsUI }));
}
} catch (error) {
if (page && !page.isClosed()) await page.screenshot({ path: path.join(run, 'failure.png'), fullPage: true }).catch(() => {});
console.error('Acceptance artifacts: ' + run);
throw error;
} finally {
if (browser) await browser.close().catch(() => {});
for (const child of [...children]) await stop(child);
if (fixture) await fixture.close();
}
Reference in New Issue View Git Blame Copy Permalink