Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
// Baseline + regression gate. This is the capability the old one-off scripts
|
||||
// never had: measured numbers are compared against a committed baseline so a
|
||||
// PR that regresses streaming/typing/mount cost fails loudly instead of
|
||||
// silently drifting.
|
||||
//
|
||||
// Every tracked metric is "lower is better" (longtask counts, frame/keystroke
|
||||
// percentiles, mount ms). A metric regresses when it exceeds
|
||||
// `baseline * (1 + tolFrac) + tolAbs`. tolAbs absorbs sub-millisecond jitter on
|
||||
// already-fast metrics so they don't false-positive.
|
||||
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
|
||||
const DEFAULT_TOLERANCE = { tolFrac: 0.25, tolAbs: 1 }
|
||||
|
||||
export function loadBaseline(path) {
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, 'utf8'))
|
||||
} catch {
|
||||
return { _meta: {}, scenarios: {} }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare a scenario's measured metrics against the baseline.
|
||||
* @returns {{ rows: Array, regressed: boolean }}
|
||||
*/
|
||||
export function compareScenario(name, measured, baseline) {
|
||||
const base = baseline.scenarios?.[name]
|
||||
const tol = { ...DEFAULT_TOLERANCE, ...(base?.tolerance ?? {}) }
|
||||
const rows = []
|
||||
let regressed = false
|
||||
|
||||
for (const [metric, value] of Object.entries(measured)) {
|
||||
if (typeof value !== 'number') {
|
||||
continue
|
||||
}
|
||||
|
||||
const baseValue = base?.metrics?.[metric]
|
||||
|
||||
if (typeof baseValue !== 'number') {
|
||||
rows.push({ metric, measured: value, baseline: null, limit: null, status: 'new' })
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
const limit = baseValue * (1 + tol.tolFrac) + tol.tolAbs
|
||||
const over = value > limit
|
||||
regressed = regressed || over
|
||||
|
||||
rows.push({
|
||||
metric,
|
||||
measured: value,
|
||||
baseline: baseValue,
|
||||
limit: Math.round(limit * 100) / 100,
|
||||
deltaPct: baseValue ? Math.round(((value - baseValue) / baseValue) * 1000) / 10 : null,
|
||||
status: over ? 'REGRESSED' : 'ok'
|
||||
})
|
||||
}
|
||||
|
||||
return { rows, regressed }
|
||||
}
|
||||
|
||||
/** Write measured metrics back as the new baseline for the given scenarios. */
|
||||
export function updateBaseline(path, results) {
|
||||
const baseline = loadBaseline(path)
|
||||
baseline.scenarios ??= {}
|
||||
|
||||
for (const { name, metrics } of results) {
|
||||
const numeric = Object.fromEntries(Object.entries(metrics).filter(([, v]) => typeof v === 'number'))
|
||||
const prev = baseline.scenarios[name] ?? {}
|
||||
baseline.scenarios[name] = { ...prev, metrics: numeric }
|
||||
}
|
||||
|
||||
baseline._meta = {
|
||||
...baseline._meta,
|
||||
updated: new Date().toISOString(),
|
||||
platform: `${process.platform}-${process.arch}`,
|
||||
node: process.version
|
||||
}
|
||||
|
||||
writeFileSync(path, `${JSON.stringify(baseline, null, 2)}\n`)
|
||||
}
|
||||
|
||||
export { DEFAULT_TOLERANCE }
|
||||
@@ -0,0 +1,202 @@
|
||||
// The one Chrome DevTools Protocol client for the desktop perf harness.
|
||||
//
|
||||
// Before this, every measure-*/profile-* script shipped its own copy-pasted
|
||||
// `CDP` class (four subtly different implementations), its own `/json` vs
|
||||
// `/json/list` target discovery, and its own Profiler ranking. Scenarios now
|
||||
// import from here so there is a single place to fix a protocol bug.
|
||||
|
||||
const DEFAULT_PORT = 9222
|
||||
|
||||
// Stable DOM hooks the renderer exposes. Centralised so a component refactor
|
||||
// updates one constant instead of a dozen scattered querySelector strings.
|
||||
export const SELECTORS = {
|
||||
composer: '[data-slot="composer-rich-input"]',
|
||||
threadViewport: '[data-slot="aui_thread-viewport"]',
|
||||
threadContent: '[data-slot="aui_thread-content"]',
|
||||
assistantMessage: '[data-slot="aui_assistant-message-root"]',
|
||||
turnPair: '[data-slot="aui_turn-pair"]',
|
||||
profileRail: '[data-slot="profile-rail"]',
|
||||
rowButton: '[data-slot="row-button"]'
|
||||
}
|
||||
|
||||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
|
||||
|
||||
/**
|
||||
* Poll the CDP HTTP endpoint until a page target is available.
|
||||
* @param {object} [opts]
|
||||
* @param {number} [opts.port] remote-debugging-port (default 9222).
|
||||
* @param {string} [opts.match] substring the target URL must contain (e.g. a dev-server port).
|
||||
* @param {number} [opts.timeoutMs] how long to wait for a target.
|
||||
*/
|
||||
export async function discoverTarget({ port = DEFAULT_PORT, match, timeoutMs = 30000 } = {}) {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
|
||||
for (;;) {
|
||||
try {
|
||||
const list = await (await fetch(`http://127.0.0.1:${port}/json/list`)).json()
|
||||
const pages = list.filter(t => t.type === 'page' && typeof t.webSocketDebuggerUrl === 'string')
|
||||
const target = match
|
||||
? pages.find(t => String(t.url).includes(match))
|
||||
: pages.find(t => String(t.url).startsWith('http')) ?? pages[0]
|
||||
|
||||
if (target) {
|
||||
return target
|
||||
}
|
||||
} catch {
|
||||
// debug port not up yet — keep polling until the deadline.
|
||||
}
|
||||
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error(`no CDP page target on :${port}${match ? ` matching "${match}"` : ''} within ${timeoutMs}ms`)
|
||||
}
|
||||
|
||||
await sleep(250)
|
||||
}
|
||||
}
|
||||
|
||||
export class CDP {
|
||||
constructor(ws) {
|
||||
this.ws = ws
|
||||
this.id = 0
|
||||
this.pending = new Map()
|
||||
this.listeners = new Map()
|
||||
}
|
||||
|
||||
static async open(url) {
|
||||
const ws = new WebSocket(url)
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
ws.addEventListener('open', resolve, { once: true })
|
||||
ws.addEventListener('error', reject, { once: true })
|
||||
})
|
||||
|
||||
const cdp = new CDP(ws)
|
||||
|
||||
ws.addEventListener('message', ev => {
|
||||
const m = JSON.parse(typeof ev.data === 'string' ? ev.data : ev.data.toString('utf8'))
|
||||
|
||||
if (m.id != null && cdp.pending.has(m.id)) {
|
||||
const { resolve, reject } = cdp.pending.get(m.id)
|
||||
cdp.pending.delete(m.id)
|
||||
|
||||
if (m.error) {
|
||||
reject(new Error(m.error.message))
|
||||
} else {
|
||||
resolve(m.result)
|
||||
}
|
||||
} else if (m.method) {
|
||||
for (const handler of cdp.listeners.get(m.method) ?? []) {
|
||||
handler(m.params)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
ws.addEventListener('close', () => {
|
||||
for (const { reject } of cdp.pending.values()) {
|
||||
reject(new Error('CDP socket closed'))
|
||||
}
|
||||
|
||||
cdp.pending.clear()
|
||||
})
|
||||
|
||||
return cdp
|
||||
}
|
||||
|
||||
/** Connect straight to a discovered target. */
|
||||
static async connect(opts) {
|
||||
const target = await discoverTarget(opts)
|
||||
|
||||
return CDP.open(target.webSocketDebuggerUrl)
|
||||
}
|
||||
|
||||
send(method, params = {}) {
|
||||
const id = ++this.id
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.pending.set(id, { resolve, reject })
|
||||
this.ws.send(JSON.stringify({ id, method, params }))
|
||||
})
|
||||
}
|
||||
|
||||
on(method, handler) {
|
||||
if (!this.listeners.has(method)) {
|
||||
this.listeners.set(method, [])
|
||||
}
|
||||
|
||||
this.listeners.get(method).push(handler)
|
||||
}
|
||||
|
||||
/** Evaluate an expression in the page and return its value (awaits promises). */
|
||||
async eval(expression) {
|
||||
const r = await this.send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true })
|
||||
|
||||
if (r.exceptionDetails) {
|
||||
throw new Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text || 'eval failed')
|
||||
}
|
||||
|
||||
return r.result.value
|
||||
}
|
||||
|
||||
close() {
|
||||
this.ws.close()
|
||||
}
|
||||
}
|
||||
|
||||
/** Assert the renderer has the dev-only `__PERF_DRIVE__` harness attached. */
|
||||
export async function requireDriver(cdp) {
|
||||
const ok = await cdp.eval('!!(window.__PERF_DRIVE__ && window.__PERF_DRIVE__.stream)')
|
||||
|
||||
if (!ok) {
|
||||
throw new Error(
|
||||
'__PERF_DRIVE__ not on window. The perf harness needs a DEV renderer ' +
|
||||
'(perf-probe.tsx is excluded from production builds). Launch with `npm run perf:serve`.'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** Type real key events into the composer, one char at a time, at `cps` chars/sec. */
|
||||
export async function typeIntoComposer(cdp, text, { cps = 15 } = {}) {
|
||||
await cdp.eval(`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
||||
if (!el) return false
|
||||
el.focus()
|
||||
const range = document.createRange()
|
||||
range.selectNodeContents(el)
|
||||
range.collapse(false)
|
||||
const sel = window.getSelection()
|
||||
sel.removeAllRanges()
|
||||
sel.addRange(range)
|
||||
return true
|
||||
})()`)
|
||||
|
||||
const intervalMs = Math.max(1, Math.round(1000 / cps))
|
||||
|
||||
for (const ch of text) {
|
||||
await cdp.send('Input.dispatchKeyEvent', { type: 'char', text: ch, unmodifiedText: ch })
|
||||
await sleep(intervalMs)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `body()` while a V8 CPU profile is recording. Returns
|
||||
* `{ result, profile }`; the caller decides whether to write the .cpuprofile.
|
||||
*/
|
||||
export async function withCpuProfile(cdp, body, { samplingIntervalUs = 100 } = {}) {
|
||||
await cdp.send('Profiler.enable')
|
||||
await cdp.send('Profiler.setSamplingInterval', { interval: samplingIntervalUs })
|
||||
await cdp.send('Profiler.start')
|
||||
|
||||
let result
|
||||
let stopped
|
||||
|
||||
try {
|
||||
result = await body()
|
||||
} finally {
|
||||
// Always stop so a scenario error can't leave the profiler running.
|
||||
stopped = await cdp.send('Profiler.stop')
|
||||
}
|
||||
|
||||
return { result, profile: stopped.profile }
|
||||
}
|
||||
|
||||
export { sleep }
|
||||
@@ -0,0 +1,423 @@
|
||||
// Connect the harness to a renderer — either an already-running debug instance
|
||||
// (`attach`) or a freshly spawned, fully isolated one (`startIsolatedInstance`).
|
||||
//
|
||||
// The isolated instance is what makes the harness self-contained and unblocks
|
||||
// the measurement that the single-instance lock used to prevent:
|
||||
// · its own --user-data-dir → its own Electron single-instance lock, so it
|
||||
// never collides with (or steals focus from) the user's running `hgui`.
|
||||
// · its own HERMES_HOME → its own backend + sessions, no shared state.
|
||||
// · its own --remote-debugging-port → a private CDP endpoint.
|
||||
// · HERMES_DESKTOP_BOOT_FAKE=1 → deterministic boot overlay.
|
||||
// The synthetic scenarios drive `$messages` directly, so no LLM credits are
|
||||
// spent regardless of the isolated backend.
|
||||
|
||||
import { spawn } from 'node:child_process'
|
||||
import { copyFileSync, existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
||||
import { createRequire } from 'node:module'
|
||||
import { homedir, tmpdir } from 'node:os'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { CDP, requireDriver, sleep } from './cdp.mjs'
|
||||
|
||||
const require = createRequire(import.meta.url)
|
||||
const DESKTOP_DIR = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..', '..')
|
||||
|
||||
async function reachable(url) {
|
||||
try {
|
||||
await fetch(url)
|
||||
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function waitFor(fn, { timeoutMs, label }) {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
if (await fn()) {
|
||||
return
|
||||
}
|
||||
|
||||
await sleep(300)
|
||||
}
|
||||
|
||||
throw new Error(`timed out after ${timeoutMs}ms waiting for ${label}`)
|
||||
}
|
||||
|
||||
// Seed an isolated HERMES_HOME with just enough config (NOT sessions) so the
|
||||
// spawned instance reaches an empty chat view instead of the onboarding wizard.
|
||||
// A separate HERMES_HOME dir means a separate gateway lock — no collision with
|
||||
// the user's running app, which keeps its own sessions DB and state.
|
||||
function seedConfigFrom(sourceHome, targetHome) {
|
||||
if (!existsSync(sourceHome)) {
|
||||
return
|
||||
}
|
||||
|
||||
for (const name of ['config.yaml', '.env', 'auth.json']) {
|
||||
const from = join(sourceHome, name)
|
||||
|
||||
if (existsSync(from)) {
|
||||
try {
|
||||
copyFileSync(from, join(targetHome, name))
|
||||
} catch {
|
||||
// best-effort — a missing file just means onboarding may appear.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the vite CLI entry via its package.json `bin` (Vite 8's `exports`
|
||||
// blocks importing `vite/bin/vite.js` directly).
|
||||
function resolveViteBin() {
|
||||
const pkgPath = require.resolve('vite/package.json')
|
||||
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8'))
|
||||
const rel = typeof pkg.bin === 'string' ? pkg.bin : pkg.bin?.vite
|
||||
|
||||
if (!rel) {
|
||||
throw new Error('could not resolve the vite CLI from vite/package.json')
|
||||
}
|
||||
|
||||
return join(dirname(pkgPath), rel)
|
||||
}
|
||||
|
||||
// Poll the perf driver's `connected()` until the gateway socket is open.
|
||||
// Returns false if the probe predates this helper or the timeout elapses.
|
||||
async function waitForConnected(cdp, timeoutMs) {
|
||||
const hasProbe = await cdp.eval('typeof window.__PERF_DRIVE__.connected === "function"')
|
||||
|
||||
if (!hasProbe) {
|
||||
return false
|
||||
}
|
||||
|
||||
const deadline = Date.now() + timeoutMs
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
if (await cdp.eval('window.__PERF_DRIVE__.connected()')) {
|
||||
return true
|
||||
}
|
||||
|
||||
await sleep(500)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
function runProcess(command, args, { env } = {}) {
|
||||
return new Promise((resolveRun, reject) => {
|
||||
const child = spawn(command, args, {
|
||||
cwd: DESKTOP_DIR,
|
||||
stdio: 'inherit',
|
||||
env: env ? { ...process.env, ...env } : process.env
|
||||
})
|
||||
child.on('error', reject)
|
||||
child.on('exit', code => (code === 0 ? resolveRun() : reject(new Error(`${command} ${args[0]} exited ${code}`))))
|
||||
})
|
||||
}
|
||||
|
||||
function runNode(scriptRelPath, args = []) {
|
||||
return runProcess(process.execPath, [join(DESKTOP_DIR, scriptRelPath), ...args])
|
||||
}
|
||||
|
||||
// Build a production renderer WITH the perf probe included (VITE_PERF_PROBE=1),
|
||||
// plus the prod electron-main bundle, so the harness can measure a real,
|
||||
// minified React build instead of the ~3x-slower dev build. Slow (a full vite
|
||||
// build); do it once, then run/attach many times.
|
||||
export async function buildProdRenderer() {
|
||||
const viteBin = resolveViteBin()
|
||||
await runProcess(process.execPath, [viteBin, 'build'], { env: { VITE_PERF_PROBE: '1' } })
|
||||
await runNode('scripts/bundle-electron-main.mjs')
|
||||
}
|
||||
|
||||
/** Attach to a renderer already listening on `port` (launched via perf:serve or with --remote-debugging-port). */
|
||||
export async function attach({ port = 9222, match } = {}) {
|
||||
const cdp = await CDP.connect({ port, match })
|
||||
await requireDriver(cdp)
|
||||
|
||||
return { cdp, teardown: () => cdp.close() }
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawn an isolated dev instance (vite + electron), wait for the perf driver,
|
||||
* and return `{ cdp, teardown, devUrl, port }`. `teardown` kills both children
|
||||
* and removes any temp dirs it created.
|
||||
*/
|
||||
// Chromium switches that stop frame-production throttling for a window that
|
||||
// isn't foregrounded (the perf window usually sits behind the IDE/terminal).
|
||||
const ANTI_THROTTLE_FLAGS = [
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--disable-backgrounding-occluded-windows',
|
||||
'--disable-features=CalculateNativeWinOcclusion'
|
||||
]
|
||||
|
||||
/**
|
||||
* Spawn an isolated instance and connect the perf driver. Two render modes:
|
||||
* · dev (default): vite dev server + dev electron-main bundle.
|
||||
* · prod (`prod: true`): a production build (call buildProdRenderer first);
|
||||
* electron loads dist/index.html — representative, minified React.
|
||||
* `coldStart: true` skips the gateway-connect wait and settle (for launch-time
|
||||
* measurement) and returns `timings` (spawn→CDP, spawn→driver) plus renderer
|
||||
* boot marks (FCP, time-to-composer).
|
||||
*/
|
||||
export async function startIsolatedInstance({
|
||||
port = 9222,
|
||||
devPort = 5174,
|
||||
prod = false,
|
||||
coldStart = false,
|
||||
hermesHome,
|
||||
userDataDir,
|
||||
seedConfig = true,
|
||||
settleMs = 2500,
|
||||
connectTimeoutMs = 90000
|
||||
} = {}) {
|
||||
const children = []
|
||||
const tempDirs = []
|
||||
|
||||
const mkTemp = prefix => {
|
||||
const dir = mkdtempSync(join(tmpdir(), prefix))
|
||||
tempDirs.push(dir)
|
||||
|
||||
return dir
|
||||
}
|
||||
|
||||
const home = hermesHome ?? mkTemp('hermes-perf-home-')
|
||||
const userData = userDataDir ?? mkTemp('hermes-perf-ud-')
|
||||
const devUrl = prod ? null : `http://127.0.0.1:${devPort}`
|
||||
|
||||
if (seedConfig && !hermesHome) {
|
||||
seedConfigFrom(join(homedir(), '.hermes'), home)
|
||||
}
|
||||
|
||||
const teardown = () => {
|
||||
for (const child of children) {
|
||||
try {
|
||||
child.kill('SIGTERM')
|
||||
} catch {
|
||||
// already gone
|
||||
}
|
||||
}
|
||||
|
||||
for (const dir of tempDirs) {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (prod) {
|
||||
// Renderer + main are expected pre-built (buildProdRenderer). Cheap to
|
||||
// re-bundle main so an isolated run always matches current source.
|
||||
await runNode('scripts/bundle-electron-main.mjs')
|
||||
} else {
|
||||
if (!(await reachable(devUrl))) {
|
||||
const viteBin = resolveViteBin()
|
||||
const vite = spawn(process.execPath, [viteBin, '--host', '127.0.0.1', '--port', String(devPort)], {
|
||||
cwd: DESKTOP_DIR,
|
||||
stdio: ['ignore', 'inherit', 'inherit']
|
||||
})
|
||||
children.push(vite)
|
||||
await waitFor(() => reachable(devUrl), { timeoutMs: 60000, label: `vite dev server on :${devPort}` })
|
||||
}
|
||||
|
||||
await runNode('scripts/bundle-electron-main.mjs', ['--dev'])
|
||||
}
|
||||
|
||||
// Isolated Electron: own --user-data-dir (single-instance lock scope) + own
|
||||
// HERMES_HOME (backend + sessions). No DEV_SERVER env in prod → dist load.
|
||||
const electronBin = require('electron')
|
||||
// NB: do NOT set HERMES_DESKTOP_BOOT_FAKE here — it injects artificial
|
||||
// per-phase sleeps into the boot overlay, which inflates cold-start timing
|
||||
// (and adds pointless startup latency to the steady-state runs). We want the
|
||||
// real boot sequence.
|
||||
const env = {
|
||||
...process.env,
|
||||
HERMES_HOME: home,
|
||||
// The app's dev-CDP resolver (electron/dev-cdp.ts) appends its own
|
||||
// remote-debugging-port switch AFTER argv, so on a non-default --port the
|
||||
// Chromium flag below loses and the instance binds 9222 anyway. The env
|
||||
// override is the supported knob — set it so --port actually wins.
|
||||
HERMES_DESKTOP_CDP_PORT: String(port),
|
||||
XCURSOR_SIZE: '24'
|
||||
}
|
||||
|
||||
if (devUrl) {
|
||||
env.HERMES_DESKTOP_DEV_SERVER = devUrl
|
||||
}
|
||||
|
||||
const spawnAt = Date.now()
|
||||
const electron = spawn(
|
||||
electronBin,
|
||||
['.', `--user-data-dir=${userData}`, `--remote-debugging-port=${port}`, ...ANTI_THROTTLE_FLAGS],
|
||||
{ cwd: DESKTOP_DIR, stdio: ['ignore', 'inherit', 'inherit'], env }
|
||||
)
|
||||
children.push(electron)
|
||||
|
||||
// Wait for the renderer + perf driver. In prod the target URL is file://,
|
||||
// so don't match on the dev port.
|
||||
let cdp = null
|
||||
let cdpAt = 0
|
||||
await waitFor(
|
||||
async () => {
|
||||
try {
|
||||
cdp = await CDP.connect({ port, match: devUrl ? String(devPort) : undefined, timeoutMs: 2000 })
|
||||
cdpAt = cdpAt || Date.now()
|
||||
|
||||
return await cdp.eval('!!(window.__PERF_DRIVE__ && window.__PERF_DRIVE__.stream)')
|
||||
} catch {
|
||||
if (cdp) {
|
||||
cdp.close()
|
||||
cdp = null
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
},
|
||||
{ timeoutMs: 120000, label: 'isolated renderer + __PERF_DRIVE__' }
|
||||
)
|
||||
const driverAt = Date.now()
|
||||
|
||||
try {
|
||||
await cdp.send('Emulation.setFocusEmulationEnabled', { enabled: true })
|
||||
} catch {
|
||||
// Older CDP / not supported — fall back to the anti-throttle flags.
|
||||
}
|
||||
|
||||
// Renderer-side boot marks (relative to its own navigation start).
|
||||
const bootMarks = await readBootMarks(cdp)
|
||||
const timings = {
|
||||
spawn_to_cdp_ms: cdpAt ? cdpAt - spawnAt : null,
|
||||
spawn_to_driver_ms: driverAt - spawnAt,
|
||||
...bootMarks
|
||||
}
|
||||
|
||||
let connected = true
|
||||
|
||||
if (!coldStart) {
|
||||
// Steady-state scenarios: wait for the gateway to connect (reconnect churn
|
||||
// contaminates frame pacing) and let residual cold-start work drain.
|
||||
connected = await waitForConnected(cdp, connectTimeoutMs)
|
||||
|
||||
if (!connected) {
|
||||
console.warn(
|
||||
`[perf] gateway did not connect within ${connectTimeoutMs}ms — ` +
|
||||
'stream/frame numbers may be inflated by reconnect churn.'
|
||||
)
|
||||
}
|
||||
|
||||
await sleep(settleMs)
|
||||
}
|
||||
|
||||
return {
|
||||
connected,
|
||||
cdp,
|
||||
devUrl,
|
||||
port,
|
||||
prod,
|
||||
timings,
|
||||
teardown: () => {
|
||||
cdp?.close()
|
||||
teardown()
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
teardown()
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
// Representative cold-start sampling. A fresh --user-data-dir means a COLD V8
|
||||
// code cache and worst-case bundle recompile every run (~+400ms measured); real
|
||||
// users reuse their profile, so a warm cache is the representative case. We reuse
|
||||
// ONE profile across runs: run 0 warms the cache (discarded), runs 1..N are the
|
||||
// warm samples. Each run steps the port so a just-killed instance can't be
|
||||
// re-attached, and we pause between runs so the single-instance lock releases.
|
||||
export async function coldStartSamples({ runs = 3, port = 9222, devPort = 5174, prod = false, warm = true } = {}) {
|
||||
const pickNumeric = timings => Object.fromEntries(Object.entries(timings).filter(([, v]) => typeof v === 'number'))
|
||||
const samples = []
|
||||
|
||||
if (warm) {
|
||||
// Shared profile across runs: run 0 warms the V8 code cache (discarded),
|
||||
// runs 1..N are the representative warm samples.
|
||||
const home = mkdtempSync(join(tmpdir(), 'hermes-perf-cold-home-'))
|
||||
const userDataDir = mkdtempSync(join(tmpdir(), 'hermes-perf-cold-ud-'))
|
||||
seedConfigFrom(join(homedir(), '.hermes'), home)
|
||||
|
||||
try {
|
||||
for (let i = 0; i <= runs; i++) {
|
||||
const inst = await startIsolatedInstance({
|
||||
port: port + i,
|
||||
devPort: devPort + i,
|
||||
prod,
|
||||
coldStart: true,
|
||||
hermesHome: home,
|
||||
userDataDir,
|
||||
seedConfig: false
|
||||
})
|
||||
|
||||
if (i > 0) {
|
||||
samples.push(pickNumeric(inst.timings))
|
||||
}
|
||||
|
||||
inst.teardown()
|
||||
await sleep(2500) // let the single-instance lock release before reuse
|
||||
}
|
||||
} finally {
|
||||
for (const dir of [home, userDataDir]) {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Worst case: a fresh profile per run → cold code cache every launch
|
||||
// (first-launch-after-install). startIsolatedInstance makes+removes its dirs.
|
||||
for (let i = 0; i < runs; i++) {
|
||||
const inst = await startIsolatedInstance({ port: port + i, devPort: devPort + i, prod, coldStart: true })
|
||||
samples.push(pickNumeric(inst.timings))
|
||||
inst.teardown()
|
||||
await sleep(2500)
|
||||
}
|
||||
}
|
||||
|
||||
return samples
|
||||
}
|
||||
|
||||
// Read First Contentful Paint + time-to-composer from the renderer, relative to
|
||||
// its navigation start (the process-spawn deltas live in `timings`).
|
||||
async function readBootMarks(cdp) {
|
||||
try {
|
||||
return await cdp.eval(`(() => {
|
||||
const paints = performance.getEntriesByType('paint')
|
||||
const fcp = paints.find(p => p.name === 'first-contentful-paint')
|
||||
const nav = performance.getEntriesByType('navigation')[0]
|
||||
const composer = document.querySelector('[data-slot="composer-rich-input"]')
|
||||
// Largest script resource ≈ the (intentionally single) renderer bundle.
|
||||
// responseEnd → the script's own decode; the eval cost shows up as the gap
|
||||
// between the bundle's responseEnd and domInteractive.
|
||||
const scripts = performance.getEntriesByType('resource').filter(r => r.initiatorType === 'script')
|
||||
const mainScript = scripts.sort((a, b) => (b.encodedBodySize || 0) - (a.encodedBodySize || 0))[0]
|
||||
const round = n => (typeof n === 'number' ? Math.round(n) : null)
|
||||
return {
|
||||
fcp_ms: fcp ? round(fcp.startTime) : null,
|
||||
dom_interactive_ms: nav ? round(nav.domInteractive) : null,
|
||||
dom_content_loaded_ms: nav ? round(nav.domContentLoadedEventEnd) : null,
|
||||
main_script_kb: mainScript ? round((mainScript.encodedBodySize || 0) / 1024) : null,
|
||||
main_script_response_end_ms: mainScript ? round(mainScript.responseEnd) : null,
|
||||
nav_to_read_ms: round(performance.now()),
|
||||
composer_present: !!composer
|
||||
}
|
||||
})()`)
|
||||
} catch {
|
||||
return { fcp_ms: null, dom_interactive_ms: null, composer_present: false }
|
||||
}
|
||||
}
|
||||
|
||||
export { DESKTOP_DIR }
|
||||
@@ -0,0 +1,89 @@
|
||||
// Shared numeric helpers for perf scenarios. Every measure-*/profile-* script
|
||||
// used to carry its own copy of these.
|
||||
|
||||
/** Nearest-rank percentile over an UNSORTED array. p in [0,1]. */
|
||||
export function percentile(values, p) {
|
||||
if (!values.length) {
|
||||
return 0
|
||||
}
|
||||
|
||||
const sorted = [...values].sort((a, b) => a - b)
|
||||
const idx = Math.min(sorted.length - 1, Math.floor(sorted.length * p))
|
||||
|
||||
return sorted[idx]
|
||||
}
|
||||
|
||||
/** min/p50/p90/p95/p99/max/mean over a sample array (rounded to 2dp). */
|
||||
export function summarize(values) {
|
||||
const round = n => Math.round(n * 100) / 100
|
||||
|
||||
if (!values.length) {
|
||||
return { n: 0, min: 0, p50: 0, p90: 0, p95: 0, p99: 0, max: 0, mean: 0 }
|
||||
}
|
||||
|
||||
const sorted = [...values].sort((a, b) => a - b)
|
||||
const mean = values.reduce((a, b) => a + b, 0) / values.length
|
||||
|
||||
return {
|
||||
n: values.length,
|
||||
min: round(sorted[0]),
|
||||
p50: round(percentile(sorted, 0.5)),
|
||||
p90: round(percentile(sorted, 0.9)),
|
||||
p95: round(percentile(sorted, 0.95)),
|
||||
p99: round(percentile(sorted, 0.99)),
|
||||
max: round(sorted[sorted.length - 1]),
|
||||
mean: round(mean)
|
||||
}
|
||||
}
|
||||
|
||||
/** Median of a numeric array (used to reduce N repeated runs to one number). */
|
||||
export function median(values) {
|
||||
return percentile(values, 0.5)
|
||||
}
|
||||
|
||||
/** Frame-interval histogram matching the buckets the stream scripts reported. */
|
||||
export function frameHistogram(frames) {
|
||||
const buckets = { '<=16.7': 0, '16.7-33': 0, '33-50': 0, '50-100': 0, '100-200': 0, '>200': 0 }
|
||||
|
||||
for (const f of frames) {
|
||||
if (f <= 16.7) buckets['<=16.7']++
|
||||
else if (f <= 33) buckets['16.7-33']++
|
||||
else if (f <= 50) buckets['33-50']++
|
||||
else if (f <= 100) buckets['50-100']++
|
||||
else if (f <= 200) buckets['100-200']++
|
||||
else buckets['>200']++
|
||||
}
|
||||
|
||||
return buckets
|
||||
}
|
||||
|
||||
/**
|
||||
* Rank functions by self-time from a V8 CPU profile (Profiler.stop output).
|
||||
* Returns the top `limit` entries as { ms, name, url, line }.
|
||||
*/
|
||||
export function cpuProfileTopSelf(profile, limit = 30) {
|
||||
const samples = profile.samples || []
|
||||
const timeDeltas = profile.timeDeltas || []
|
||||
const nodes = new Map(profile.nodes.map(n => [n.id, n]))
|
||||
const selfUs = new Map()
|
||||
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
const id = samples[i]
|
||||
selfUs.set(id, (selfUs.get(id) || 0) + (timeDeltas[i] ?? 0))
|
||||
}
|
||||
|
||||
return [...selfUs.entries()]
|
||||
.map(([id, us]) => {
|
||||
const cf = nodes.get(id)?.callFrame || {}
|
||||
|
||||
return {
|
||||
ms: us / 1000,
|
||||
name: cf.functionName || '(anonymous)',
|
||||
url: String(cf.url || '').slice(-70),
|
||||
line: cf.lineNumber
|
||||
}
|
||||
})
|
||||
.filter(x => !/\(root\)|\(idle\)|\(garbage collector\)|\(program\)/.test(x.name))
|
||||
.sort((a, b) => b.ms - a.ms)
|
||||
.slice(0, limit)
|
||||
}
|
||||
Reference in New Issue
Block a user