Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
// Cold start — launch → renderer → interactive. Unlike the other scenarios this
|
||||
// measures the LAUNCH itself, so it can't run against an already-up instance:
|
||||
// the runner spawns a fresh isolated instance per run (requires --spawn) and
|
||||
// reads the timings/boot-marks the launcher captures. Registered here so it's a
|
||||
// known name with a baseline entry; the actual measurement lives in run.mjs.
|
||||
//
|
||||
// Metrics (lower is better):
|
||||
// spawn_to_cdp_ms process spawn → CDP page target reachable (electron/V8 up)
|
||||
// spawn_to_driver_ms process spawn → renderer mounted + perf driver present
|
||||
// fcp_ms renderer nav start → first contentful paint
|
||||
export default {
|
||||
name: 'cold-start',
|
||||
tier: 'cold',
|
||||
description: 'Launch → first paint → interactive (fresh spawn per run).',
|
||||
run() {
|
||||
throw new Error('cold-start is measured by the runner via fresh spawns; use `--spawn`.')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
// Time-to-first-token — Enter → first assistant token painted. The latency an
|
||||
// agent app is uniquely judged on, spanning the desktop submit path AND the
|
||||
// backend/agent-loop first-token time. Backend tier: fires a REAL prompt, needs
|
||||
// a live backend (and credits). Report-only.
|
||||
//
|
||||
// node scripts/perf/run.mjs first-token --spawn --prompt "hi"
|
||||
|
||||
import { SELECTORS, sleep, typeIntoComposer } from '../lib/cdp.mjs'
|
||||
import { summarize } from '../lib/stats.mjs'
|
||||
|
||||
export default {
|
||||
name: 'first-token',
|
||||
tier: 'backend',
|
||||
description: 'Enter → first assistant token painted (real backend).',
|
||||
async run(cdp, opts = {}) {
|
||||
const rounds = Number(opts.rounds ?? 3)
|
||||
const prompt = opts.prompt ?? 'reply with a single short sentence'
|
||||
const timeoutMs = Number(opts.timeoutMs ?? 60000)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const firstTokens = []
|
||||
|
||||
for (let i = 0; i < rounds; i++) {
|
||||
const baseText = await cdp.eval(`(() => {
|
||||
const a = document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)})
|
||||
return a.length ? a[a.length - 1].textContent.length : 0
|
||||
})()`)
|
||||
const baseCount = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`)
|
||||
|
||||
await typeIntoComposer(cdp, `${prompt} (${i})`, { cps: 60 })
|
||||
const submitAt = Date.now()
|
||||
await cdp.eval(`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
||||
el && el.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', bubbles: true, cancelable: true }))
|
||||
})()`)
|
||||
|
||||
const deadline = Date.now() + timeoutMs
|
||||
let firstTokenMs = null
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
await sleep(25)
|
||||
const grown = await cdp.eval(`(() => {
|
||||
const a = document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)})
|
||||
if (a.length > ${baseCount}) return true
|
||||
return a.length ? a[a.length - 1].textContent.length > ${baseText} : false
|
||||
})()`)
|
||||
|
||||
if (grown) {
|
||||
firstTokenMs = Date.now() - submitAt
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (firstTokenMs !== null) {
|
||||
firstTokens.push(firstTokenMs)
|
||||
}
|
||||
|
||||
// Let the turn finish before the next round.
|
||||
const turnDeadline = Date.now() + timeoutMs
|
||||
while (Date.now() < turnDeadline) {
|
||||
await sleep(250)
|
||||
const busy = await cdp.eval(`!!document.querySelector('[data-status="running"], [data-busy="true"]')`)
|
||||
|
||||
if (!busy) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(500)
|
||||
}
|
||||
|
||||
if (!firstTokens.length) {
|
||||
throw new Error('no first token observed — is a backend with credits connected?')
|
||||
}
|
||||
|
||||
const s = summarize(firstTokens)
|
||||
|
||||
return {
|
||||
metrics: { first_token_p50_ms: s.p50, first_token_p95_ms: s.p95 },
|
||||
detail: { rounds, samples: firstTokens, summary: s }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
// What does the app cost while a turn is running but NOTHING is arriving?
|
||||
//
|
||||
// The user-visible symptom: with a thread spinning, resizing the sidebar or
|
||||
// typing in the composer feels slow. That is not streaming cost — the stream
|
||||
// is idle. It is the app re-rendering on its own, competing with the
|
||||
// interaction for the main thread.
|
||||
//
|
||||
// This scenario holds N tiles in a busy state, pushes NO tokens, and measures:
|
||||
// - idle_commits_per_s the renderer's self-inflicted commit rate
|
||||
// - drag_fps fps while dragging the sidebar splitter
|
||||
// - type_fps fps while typing in the composer
|
||||
//
|
||||
// A perfectly idle app scores 0 idle commits and pins both interactions at the
|
||||
// display's refresh rate. Every idle commit is main-thread time stolen from an
|
||||
// interaction the user can feel.
|
||||
//
|
||||
// node scripts/perf/run.mjs idle-cost --spawn [--tiles 5] [--seconds 6]
|
||||
|
||||
import { sleep } from '../lib/cdp.mjs'
|
||||
|
||||
/** Seed `tiles` busy session tiles. Same publish path as `multitab` /
|
||||
* `render-churn`, but the driver never runs — the turn just stays open. */
|
||||
const setup = (tiles, seedTurns) => `
|
||||
(() => {
|
||||
const hook = window.__HERMES_SESSION_TILES__
|
||||
if (!hook) return 'no-hook'
|
||||
if (!window.__RENDER_COUNTS__) return 'no-render-counter'
|
||||
|
||||
const turn = (sid, i) => ([
|
||||
{ id: sid + '-u' + i, role: 'user', timestamp: Date.now(),
|
||||
parts: [{ type: 'text', text: 'Question ' + i + ' about the diff.' }] },
|
||||
{ id: sid + '-a' + i, role: 'assistant', timestamp: Date.now(), pending: false,
|
||||
parts: [{ type: 'text', text: '## Finding ' + i + '\\n\\nThe handler swallows the rejection.\\n\\n- Point one.\\n- Point two.\\n' }] }
|
||||
])
|
||||
|
||||
window.__IDLE__ = { ids: [] }
|
||||
for (let n = 1; n <= ${tiles}; n++) {
|
||||
const sid = 'idle-tile-' + n
|
||||
const rid = 'idle-rt-' + n
|
||||
const messages = []
|
||||
for (let i = 0; i < ${seedTurns}; i++) messages.push(...turn(sid, i))
|
||||
// An OPEN assistant message: the turn is running, but no tokens arrive.
|
||||
messages.push({ id: sid + '-stream', role: 'assistant', timestamp: Date.now(), pending: true,
|
||||
parts: [{ type: 'text', text: 'Working on it.' }] })
|
||||
|
||||
window.__IDLE__.ids.push({ sid, rid })
|
||||
hook.open(sid, 'center')
|
||||
hook.patch(sid, { runtimeId: rid })
|
||||
hook.publish(rid, {
|
||||
storedSessionId: sid, messages, branch: '', cwd: '', model: '', provider: '',
|
||||
reasoningEffort: '', serviceTier: '', fast: false, yolo: false, personality: '',
|
||||
busy: true, awaitingResponse: false, streamId: sid + '-stream', sawAssistantPayload: true,
|
||||
pendingBranchGroup: null, interrupted: false, interimBoundaryPending: false,
|
||||
needsInput: false, turnStartedAt: Date.now(), usage: null
|
||||
})
|
||||
}
|
||||
return 'ok'
|
||||
})()
|
||||
`
|
||||
|
||||
const reveal = sid => `window.__HERMES_LAYOUT_TREE__.reveal(${JSON.stringify(`session-tile:${sid}`)})`
|
||||
|
||||
/** Measure the app's self-inflicted commit rate with nothing happening. */
|
||||
const idleCost = seconds => `
|
||||
(async () => {
|
||||
const rc = window.__RENDER_COUNTS__
|
||||
rc.start()
|
||||
const t0 = performance.now()
|
||||
await new Promise(r => setTimeout(r, ${seconds} * 1000))
|
||||
const elapsed = (performance.now() - t0) / 1000
|
||||
rc.stop()
|
||||
return JSON.stringify({
|
||||
elapsed,
|
||||
commits: rc.commits(),
|
||||
top: rc.report(12),
|
||||
owners: rc.report(300).filter(r => r.stateChanged > 0 && r.propsChanged === 0).slice(0, 10)
|
||||
})
|
||||
})()
|
||||
`
|
||||
|
||||
/** Record frame pacing across an interaction driven from the page.
|
||||
*
|
||||
* The gesture body drives itself on requestAnimationFrame, so it IS the frame
|
||||
* clock — timing is taken from those same callbacks rather than a second,
|
||||
* independent rAF ticker. Running two rAF consumers made the observer's
|
||||
* deltas count the driver's frames as well as the app's and reported ~3fps
|
||||
* where the interaction actually ran at ~23fps. `frames` is filled by the
|
||||
* body via `__MARK__`.
|
||||
*
|
||||
* `record` MUST be false for any fps number you intend to believe: the render
|
||||
* counter walks the whole fiber tree on every commit, so recording during a
|
||||
* gesture measures the instrumentation as much as the app. Attribution and
|
||||
* timing therefore run as two separate passes. */
|
||||
const withFrames = (body, record = false) => `
|
||||
(async () => {
|
||||
const rc = window.__RENDER_COUNTS__
|
||||
${record ? 'rc.start()' : ''}
|
||||
const frames = []
|
||||
let last = performance.now()
|
||||
// The body calls this once per frame it drives.
|
||||
const __MARK__ = () => {
|
||||
const now = performance.now()
|
||||
frames.push(now - last)
|
||||
last = now
|
||||
}
|
||||
${body}
|
||||
${record ? 'rc.stop()' : ''}
|
||||
const total = frames.reduce((a, b) => a + b, 0)
|
||||
const sorted = [...frames].sort((a, b) => a - b)
|
||||
const pct = p => sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))] : 0
|
||||
return JSON.stringify({
|
||||
fps: total ? (frames.length / total) * 1000 : 0,
|
||||
p95: pct(0.95),
|
||||
worst: sorted.length ? sorted[sorted.length - 1] : 0,
|
||||
slow33: frames.filter(f => f > 33).length,
|
||||
n: frames.length,
|
||||
commits: ${record ? 'rc.commits()' : '0'},
|
||||
top: ${record ? 'rc.report(10)' : '[]'}
|
||||
})
|
||||
})()
|
||||
`
|
||||
|
||||
/** Drag the sidebar splitter — the resize symptom.
|
||||
* Sweeps monotonically (an oscillation nets to zero and can clamp to a no-op),
|
||||
* and reports how far it actually moved so a drag that silently did nothing
|
||||
* shows up as `dragMoved: 0` instead of a confident wrong number. */
|
||||
const DRAG = withFrames(`
|
||||
const handle = document.querySelector('[role="separator"]')
|
||||
window.__DRAG_TARGET__ = handle ? 'separator' : 'none'
|
||||
window.__DRAG_MOVED__ = 0
|
||||
if (handle) {
|
||||
const box = handle.getBoundingClientRect()
|
||||
const y = box.top + box.height / 2
|
||||
const x0 = box.left + box.width / 2
|
||||
let x = x0
|
||||
const opts = { bubbles: true, cancelable: true, pointerId: 1, pointerType: 'mouse', isPrimary: true, button: 0, buttons: 1 }
|
||||
handle.dispatchEvent(new PointerEvent('pointerdown', { ...opts, clientX: x, clientY: y }))
|
||||
// Out 60px then back — a real gesture, with a net displacement at the peak.
|
||||
for (let i = 0; i < 30; i++) {
|
||||
x += 2
|
||||
window.dispatchEvent(new PointerEvent('pointermove', { ...opts, clientX: x, clientY: y }))
|
||||
await new Promise(r => requestAnimationFrame(r))
|
||||
__MARK__()
|
||||
}
|
||||
window.__DRAG_MOVED__ = Math.round(x - x0)
|
||||
for (let i = 0; i < 30; i++) {
|
||||
x -= 2
|
||||
window.dispatchEvent(new PointerEvent('pointermove', { ...opts, clientX: x, clientY: y }))
|
||||
await new Promise(r => requestAnimationFrame(r))
|
||||
__MARK__()
|
||||
}
|
||||
window.dispatchEvent(new PointerEvent('pointerup', { ...opts, buttons: 0, clientX: x, clientY: y }))
|
||||
} else {
|
||||
await new Promise(r => setTimeout(r, 1000))
|
||||
}
|
||||
`)
|
||||
|
||||
/** Type into the composer — the keystroke symptom. */
|
||||
const TYPE = withFrames(`
|
||||
const el = document.querySelector('[contenteditable="true"], textarea')
|
||||
window.__TYPE_TARGET__ = el ? (el.tagName.toLowerCase()) : 'none'
|
||||
if (el) {
|
||||
el.focus()
|
||||
for (let i = 0; i < 40; i++) {
|
||||
const ch = 'performance testing '[i % 20]
|
||||
el.dispatchEvent(new KeyboardEvent('keydown', { bubbles: true, key: ch }))
|
||||
if (el.tagName === 'TEXTAREA') {
|
||||
el.value += ch
|
||||
el.dispatchEvent(new Event('input', { bubbles: true }))
|
||||
} else {
|
||||
el.textContent += ch
|
||||
el.dispatchEvent(new InputEvent('input', { bubbles: true, data: ch, inputType: 'insertText' }))
|
||||
}
|
||||
el.dispatchEvent(new KeyboardEvent('keyup', { bubbles: true, key: ch }))
|
||||
// Wait for the frame this keystroke produces, then mark it — same clock
|
||||
// discipline as DRAG, so typing fps is comparable to drag fps.
|
||||
await new Promise(r => requestAnimationFrame(r))
|
||||
__MARK__()
|
||||
await new Promise(r => setTimeout(r, 25))
|
||||
}
|
||||
} else {
|
||||
await new Promise(r => setTimeout(r, 1000))
|
||||
}
|
||||
`)
|
||||
|
||||
const CLEANUP = `
|
||||
(() => {
|
||||
if (window.__IDLE__) {
|
||||
for (const { sid, rid } of window.__IDLE__.ids) {
|
||||
const states = window.__HERMES_SESSION_TILES__.states()
|
||||
window.__HERMES_SESSION_TILES__.publish(rid, { ...states[rid], busy: false, streamId: null })
|
||||
window.__HERMES_SESSION_TILES__.close(sid)
|
||||
}
|
||||
window.__IDLE__ = null
|
||||
}
|
||||
window.__RENDER_COUNTS__.clear()
|
||||
return 'cleaned'
|
||||
})()
|
||||
`
|
||||
|
||||
const round = (n, places = 1) => Math.round(n * 10 ** places) / 10 ** places
|
||||
|
||||
export default {
|
||||
name: 'idle-cost',
|
||||
// NOT 'ci': the drag fps this reports (~0.6fps, p95 814ms) contradicts a
|
||||
// direct single-clock probe of the same gesture on the same build (57fps),
|
||||
// and I could not reconcile the two — ruled out sash selection, tile setup,
|
||||
// counter residue, and a 20s soak. Its RENDER attribution and idle commit
|
||||
// rate are trustworthy and are what this scenario is for; the interaction
|
||||
// fps is reported for investigation, not gated on, until that is explained.
|
||||
tier: 'report',
|
||||
description: 'Busy-but-silent tiles: idle commit rate, and fps while resizing / typing.',
|
||||
async run(cdp, opts = {}) {
|
||||
const tiles = Number(opts.tiles ?? 5)
|
||||
const seedTurns = Number(opts.turns ?? 20)
|
||||
const seconds = Number(opts.seconds ?? 6)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const ok = await cdp.eval(setup(tiles, seedTurns))
|
||||
|
||||
if (ok !== 'ok') {
|
||||
throw new Error(`idle-cost setup failed (${ok}) — needs a dev renderer with src/debug installed.`)
|
||||
}
|
||||
|
||||
for (let n = 1; n <= tiles; n++) {
|
||||
await cdp.eval(reveal(`idle-tile-${n}`))
|
||||
await sleep(300)
|
||||
}
|
||||
|
||||
await sleep(1500)
|
||||
|
||||
const idle = JSON.parse(await cdp.eval(idleCost(seconds)))
|
||||
const drag = JSON.parse(await cdp.eval(DRAG))
|
||||
const dragTarget = await cdp.eval('window.__DRAG_TARGET__ || "unknown"')
|
||||
const dragMoved = await cdp.eval('window.__DRAG_MOVED__ ?? 0')
|
||||
const type = JSON.parse(await cdp.eval(TYPE))
|
||||
const typeTarget = await cdp.eval('window.__TYPE_TARGET__ || "unknown"')
|
||||
|
||||
await cdp.eval(CLEANUP)
|
||||
|
||||
if (dragTarget === 'none') {
|
||||
throw new Error('idle-cost: no [role="separator"] sash found — the drag measured nothing.')
|
||||
}
|
||||
|
||||
if (typeTarget === 'none') {
|
||||
throw new Error('idle-cost: no composer found — the typing pass measured nothing.')
|
||||
}
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
// Commits per second with a turn open and nothing arriving. Should be 0.
|
||||
idle_commits_per_s: round(idle.commits / idle.elapsed),
|
||||
idle_renders: idle.top.reduce((a, r) => a + r.renders, 0),
|
||||
// Interaction smoothness while that churn competes for the main thread.
|
||||
// Reported as a deficit from 60fps so "lower is better" matches the
|
||||
// baseline gate's direction.
|
||||
drag_fps_deficit: round(Math.max(0, 60 - drag.fps)),
|
||||
drag_slow_frames: drag.slow33,
|
||||
type_fps_deficit: round(Math.max(0, 60 - type.fps)),
|
||||
type_slow_frames: type.slow33
|
||||
},
|
||||
detail: {
|
||||
tiles,
|
||||
dragTarget,
|
||||
dragMoved,
|
||||
idleSeconds: round(idle.elapsed),
|
||||
dragFps: round(drag.fps),
|
||||
dragP95: round(drag.p95),
|
||||
dragWorst: round(drag.worst),
|
||||
typeFps: round(type.fps),
|
||||
typeP95: round(type.p95),
|
||||
typeWorst: round(type.worst),
|
||||
// Components whose OWN state changed with no prop change: the roots.
|
||||
idleOwners: idle.owners,
|
||||
idleTop: idle.top,
|
||||
dragCommits: drag.commits,
|
||||
dragTop: drag.top,
|
||||
typeCommits: type.commits,
|
||||
typeTop: type.top
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
// Scenario registry. Add a scenario module here and it's automatically
|
||||
// available to the runner, the default suite (tier 'ci'), and the baseline gate.
|
||||
|
||||
import coldStart from './cold-start.mjs'
|
||||
import firstToken from './first-token.mjs'
|
||||
import idleCost from './idle-cost.mjs'
|
||||
import keystroke from './keystroke.mjs'
|
||||
import multitab from './multitab.mjs'
|
||||
import profileSwitch from './profile-switch.mjs'
|
||||
import renderChurn from './render-churn.mjs'
|
||||
import rightPane from './right-pane.mjs'
|
||||
import sessionLoad from './session-load.mjs'
|
||||
import sessionSwitch from './session-switch.mjs'
|
||||
import stream from './stream.mjs'
|
||||
import streamHistory from './stream-history.mjs'
|
||||
import submit from './submit.mjs'
|
||||
import transcript from './transcript.mjs'
|
||||
|
||||
export const SCENARIOS = {
|
||||
[stream.name]: stream,
|
||||
[streamHistory.name]: streamHistory,
|
||||
[keystroke.name]: keystroke,
|
||||
[transcript.name]: transcript,
|
||||
[multitab.name]: multitab,
|
||||
[renderChurn.name]: renderChurn,
|
||||
[rightPane.name]: rightPane,
|
||||
[idleCost.name]: idleCost,
|
||||
[coldStart.name]: coldStart,
|
||||
[firstToken.name]: firstToken,
|
||||
[submit.name]: submit,
|
||||
[sessionLoad.name]: sessionLoad,
|
||||
[sessionSwitch.name]: sessionSwitch,
|
||||
[profileSwitch.name]: profileSwitch
|
||||
}
|
||||
|
||||
/** Scenarios safe to run with no LLM credits / no live backend — the default suite. */
|
||||
export const CI_SCENARIOS = Object.values(SCENARIOS)
|
||||
.filter(s => s.tier === 'ci')
|
||||
.map(s => s.name)
|
||||
@@ -0,0 +1,99 @@
|
||||
// Composer input latency — keystroke → next paint. Subsumes measure-latency,
|
||||
// profile-typing, and leak-typing. This is the most-felt latency in a chat app
|
||||
// (users type constantly) and nothing measured it against a baseline before.
|
||||
//
|
||||
// Each synthetic char records the time from dispatch to the first rAF after the
|
||||
// composer mutates (a paint proxy). Metrics are p50/p95/p99 and the count of
|
||||
// keystrokes that missed a 16ms frame.
|
||||
|
||||
import { SELECTORS, sleep } from '../lib/cdp.mjs'
|
||||
import { percentile } from '../lib/stats.mjs'
|
||||
|
||||
const INSTALL = `
|
||||
(() => {
|
||||
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
||||
if (!el) return false
|
||||
el.focus()
|
||||
const range = document.createRange()
|
||||
range.selectNodeContents(el)
|
||||
range.collapse(false)
|
||||
const sel = window.getSelection()
|
||||
sel.removeAllRanges()
|
||||
sel.addRange(range)
|
||||
window.__KEY__ = { samples: [], pending: null }
|
||||
const obs = new MutationObserver(() => {
|
||||
const start = window.__KEY__.pending
|
||||
if (start === null) return
|
||||
window.__KEY__.pending = null
|
||||
requestAnimationFrame(() => window.__KEY__.samples.push(performance.now() - start))
|
||||
})
|
||||
obs.observe(el, { childList: true, subtree: true, characterData: true })
|
||||
window.__KEY__.obs = obs
|
||||
return true
|
||||
})()
|
||||
`
|
||||
|
||||
const CLEAR = `
|
||||
(() => {
|
||||
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
||||
if (el) { el.innerHTML = ''; el.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContentBackward' })) }
|
||||
window.__KEY__ && window.__KEY__.obs && window.__KEY__.obs.disconnect()
|
||||
})()
|
||||
`
|
||||
|
||||
const SENTENCE =
|
||||
'the quick brown fox jumps over the lazy dog while typing into this composer, which should feel instant. '
|
||||
|
||||
export default {
|
||||
name: 'keystroke',
|
||||
tier: 'ci',
|
||||
description: 'Composer keystroke → paint latency while idle.',
|
||||
async run(cdp, opts = {}) {
|
||||
const chars = Number(opts.chars ?? 120)
|
||||
const cps = Number(opts.cps ?? 15)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const installed = await cdp.eval(INSTALL)
|
||||
|
||||
if (!installed) {
|
||||
throw new Error(`composer not found (${SELECTORS.composer}); is a chat view open?`)
|
||||
}
|
||||
|
||||
let text = ''
|
||||
|
||||
while (text.length < chars) {
|
||||
text += SENTENCE
|
||||
}
|
||||
|
||||
text = text.slice(0, chars)
|
||||
const intervalMs = Math.max(1, Math.round(1000 / cps))
|
||||
const start = Date.now()
|
||||
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
await cdp.eval('window.__KEY__.pending = performance.now()')
|
||||
await cdp.send('Input.dispatchKeyEvent', { type: 'char', text: text[i], unmodifiedText: text[i] })
|
||||
const wait = start + (i + 1) * intervalMs - Date.now()
|
||||
|
||||
if (wait > 0) {
|
||||
await sleep(wait)
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(300)
|
||||
const samples = await cdp.eval('window.__KEY__.samples')
|
||||
await cdp.eval(CLEAR)
|
||||
|
||||
const round = n => Math.round(n * 10) / 10
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
keystroke_p50_ms: round(percentile(samples, 0.5)),
|
||||
keystroke_p95_ms: round(percentile(samples, 0.95)),
|
||||
keystroke_p99_ms: round(percentile(samples, 0.99)),
|
||||
keystroke_slow_16: samples.filter(s => s > 16).length
|
||||
},
|
||||
detail: { n: samples.length, typed: text.length }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
// Multi-tab working sessions: N session tiles stacked as tabs in the main
|
||||
// zone, EVERY tab mounted (keep-alive), all streaming concurrently — the
|
||||
// "5 tabs doing PR review" workload. Measures frame pacing + longtasks while
|
||||
// the whole stack streams, which is where multitab renderers crawl.
|
||||
//
|
||||
// --zones M splits the tiles across M VISIBLE split zones (a 2×2 grid for 4)
|
||||
// instead of one tab stack — the "4 tiles with 4 sessions each" workload,
|
||||
// where M transcripts stream on screen at once and the rest are mounted
|
||||
// keep-alive tabs behind them. --streaming S caps how many sessions are
|
||||
// actually mid-turn (zone leaders first, so S=zones means "every visible
|
||||
// transcript streams, every hidden tab idles"); the rest sit settled.
|
||||
// --sessions N seeds a populated recents list (a lived-in sessions DB).
|
||||
// --turns N sets transcript depth per tile (long sessions), and --tools makes
|
||||
// every transcript an AGENT session: seeded turns carry settled tool rounds,
|
||||
// and the live stream opens/completes tool calls between text chunks.
|
||||
//
|
||||
// Drives the real pipeline synthetically (no backend, no credits): each tick
|
||||
// routes one delta per streaming session through `hook.update` — the same
|
||||
// wiring-cache write (journal + publish + view sync) the gateway's delta
|
||||
// flush performs — via the __HERMES_SESSION_TILES__ hook.
|
||||
//
|
||||
// node scripts/perf/run.mjs multitab --spawn [--tiles 5] [--tokens 240]
|
||||
// node scripts/perf/run.mjs multitab --spawn --tiles 16 --zones 4 --sessions 300
|
||||
|
||||
import { sleep } from '../lib/cdp.mjs'
|
||||
import { frameHistogram, percentile } from '../lib/stats.mjs'
|
||||
|
||||
// Same recorder pattern as stream.mjs (generation-guarded rAF + longtasks).
|
||||
const RECORDERS = `
|
||||
(() => {
|
||||
window.__FT_GEN__ = (window.__FT_GEN__ || 0) + 1
|
||||
const ftGen = window.__FT_GEN__
|
||||
window.__FT__ = { times: [], stop: false }
|
||||
let last = performance.now()
|
||||
const tick = () => {
|
||||
if (window.__FT_GEN__ !== ftGen || window.__FT__.stop) return
|
||||
const now = performance.now()
|
||||
window.__FT__.times.push(now - last)
|
||||
last = now
|
||||
requestAnimationFrame(tick)
|
||||
}
|
||||
requestAnimationFrame(tick)
|
||||
|
||||
window.__LT__ = { entries: [], stop: false }
|
||||
try {
|
||||
const po = new PerformanceObserver((list) => {
|
||||
if (window.__LT__.stop) return
|
||||
for (const e of list.getEntries()) window.__LT__.entries.push({ duration: e.duration, startTime: e.startTime })
|
||||
})
|
||||
po.observe({ entryTypes: ['longtask'] })
|
||||
window.__LT__.po = po
|
||||
} catch {}
|
||||
return 'armed'
|
||||
})()
|
||||
`
|
||||
|
||||
const COLLECT = `
|
||||
(() => {
|
||||
window.__FT__.stop = true
|
||||
window.__LT__.stop = true
|
||||
try { window.__LT__.po && window.__LT__.po.disconnect() } catch {}
|
||||
return JSON.stringify({ frames: window.__FT__.times, longtasks: window.__LT__.entries })
|
||||
})()
|
||||
`
|
||||
|
||||
/** Page-side setup: open `tiles` session tiles — one tab stack in the main
|
||||
* zone (zones=1), or spread across `zones` visible splits (a 2×2 grid for 4)
|
||||
* — bind fake runtime ids, and seed each with a realistic transcript.
|
||||
*
|
||||
* States are written through `hook.update` — the REAL gateway write path
|
||||
* (wiring cache + in-flight journal + publish + view sync). Driving
|
||||
* `hook.publish` alone under-models a stream: it skips the journal and the
|
||||
* cache, which is exactly where multi-session cost used to hide. */
|
||||
const setup = (tiles, seedTurns, streamSeed, zones, seedSessions, streaming, dead, tools) => `
|
||||
(() => {
|
||||
const hook = window.__HERMES_SESSION_TILES__
|
||||
if (!hook) return 'no-hook'
|
||||
if (!hook.update) return 'no-update-hook'
|
||||
|
||||
// A settled tool call the way the gateway stores one: streamed args (kept
|
||||
// as argsText too) and a result blob. Real agent transcripts are MOSTLY
|
||||
// these — a long session is hundreds of terminal/read_file/patch rounds.
|
||||
const toolPart = (sid, i, k) => {
|
||||
const args = { command: 'rg -n "handler" src/module-' + i + ' | head -40', background: false }
|
||||
return {
|
||||
type: 'tool-call', toolCallId: sid + '-t' + i + '-' + k, toolName: k % 2 ? 'read_file' : 'terminal',
|
||||
args, argsText: JSON.stringify(args),
|
||||
result: JSON.stringify({ success: true, output: Array.from({ length: 18 },
|
||||
(_, l) => 'src/module-' + i + '.ts:' + (l * 7 + 3) + ': const handler = wrap(ctx, retry)').join('\\n') })
|
||||
}
|
||||
}
|
||||
|
||||
const turn = (sid, i) => {
|
||||
const answer = { id: sid + '-a' + i, role: 'assistant', timestamp: Date.now(), pending: false,
|
||||
parts: [{ type: 'text', text: [
|
||||
'## Finding ' + i, '',
|
||||
'The handler swallows the rejection. Key points for hunk \\\`' + i + '\\\`:', '',
|
||||
'- The catch block drops the original error.',
|
||||
'- Retries are unbounded — see [the loop](https://example.com/loop).', '',
|
||||
'\\\`\\\`\\\`ts',
|
||||
'async function retry' + i + '(fn: () => Promise<void>) {',
|
||||
' for (;;) { try { return await fn() } catch {} }',
|
||||
'}',
|
||||
'\\\`\\\`\\\`', '',
|
||||
'| path | covered |', '|---|---|', '| happy | yes |', '| error | no |', ''
|
||||
].join('\\n') }] }
|
||||
const rows = [
|
||||
{ id: sid + '-u' + i, role: 'user', timestamp: Date.now(),
|
||||
parts: [{ type: 'text', text: 'Review question ' + i + ': does the diff in module ' + i + ' handle the error path?' }] }
|
||||
]
|
||||
// Agent work turn (--tools): two tool rounds before the answer, the
|
||||
// shape run_conversation actually produces.
|
||||
if (${tools}) {
|
||||
rows.push({ id: sid + '-w' + i, role: 'assistant', timestamp: Date.now(), pending: false,
|
||||
parts: [{ type: 'text', text: 'Checking module ' + i + '.' }, toolPart(sid, i, 0), toolPart(sid, i, 1)] })
|
||||
}
|
||||
rows.push(answer)
|
||||
return rows
|
||||
}
|
||||
|
||||
const state = (sid, rid, isStreaming) => {
|
||||
const messages = []
|
||||
for (let i = 0; i < ${seedTurns}; i++) messages.push(...turn(sid, i))
|
||||
// Streaming tail the driver grows (--code seeds an open fence); a
|
||||
// non-streaming session sits settled — open, mounted, mid-nothing.
|
||||
if (isStreaming) {
|
||||
messages.push({ id: sid + '-stream', role: 'assistant', timestamp: Date.now(), pending: true,
|
||||
parts: [{ type: 'text', text: ${JSON.stringify(streamSeed)} }] })
|
||||
}
|
||||
return {
|
||||
storedSessionId: sid, messages, branch: '', cwd: '', model: '', provider: '',
|
||||
reasoningEffort: '', serviceTier: '', fast: false, yolo: false, personality: '',
|
||||
busy: isStreaming, awaitingResponse: false,
|
||||
streamId: isStreaming ? sid + '-stream' : null, sawAssistantPayload: true,
|
||||
pendingBranchGroup: null, interrupted: false, interimBoundaryPending: false,
|
||||
needsInput: false, turnStartedAt: isStreaming ? Date.now() : null, usage: null
|
||||
}
|
||||
}
|
||||
|
||||
// A populated recents list (--sessions): every store publish re-runs the
|
||||
// busy/attention/draft projections against it, so an empty list hides
|
||||
// that scaling. Restored by CLEANUP.
|
||||
if (${seedSessions} > 0) {
|
||||
window.__MT_SAVED_SESSIONS__ = hook.sessions()
|
||||
const rows = []
|
||||
for (let i = 0; i < ${seedSessions}; i++) {
|
||||
rows.push({
|
||||
id: 'perf-row-' + i, title: 'Seeded session ' + i, ended_at: null,
|
||||
input_tokens: 1200, output_tokens: 800, is_active: false,
|
||||
last_active: Date.now() - i * 60000, message_count: 12,
|
||||
model: 'hermes-4', preview: 'seeded row', cwd: '/tmp/proj-' + (i % 7)
|
||||
})
|
||||
}
|
||||
hook.seedSessions(rows)
|
||||
}
|
||||
|
||||
// Leaked residue (--dead): sessions that ran with no surface referencing
|
||||
// them and then settled — what a day of opening and closing tiles
|
||||
// accumulates. Modeled on the real path (insert while busy, then the
|
||||
// settle publish) so publish-time eviction, where present, engages.
|
||||
// CLEANUP drops whatever survives, for builds without eviction.
|
||||
window.__MT_DEAD__ = []
|
||||
for (let d = 0; d < ${dead}; d++) {
|
||||
const sid = 'perf-dead-' + d
|
||||
const rid = 'perf-dead-rt-' + d
|
||||
window.__MT_DEAD__.push(rid)
|
||||
const settled = state(sid, rid, false)
|
||||
hook.publish(rid, { ...settled, busy: true })
|
||||
hook.publish(rid, settled)
|
||||
}
|
||||
|
||||
// Zone leaders open as visible splits (right of the workspace, then
|
||||
// subdividing that column into a grid); followers stack as tabs into
|
||||
// their zone. zones=1 keeps the classic one-stack workload.
|
||||
const perZone = Math.ceil(${tiles} / ${zones})
|
||||
const leaders = []
|
||||
|
||||
// Streaming slots go to zone LEADERS first (rank orders round-robin across
|
||||
// zones), so --streaming ${'$'}{zones} means "every VISIBLE transcript streams,
|
||||
// every hidden tab idles" — the split the all-vs-visible snapshots diff.
|
||||
window.__MT__ = { ids: [], leaders, streaming: [], timer: null }
|
||||
for (let n = 1; n <= ${tiles}; n++) {
|
||||
const sid = 'perf-tile-' + n
|
||||
const rid = 'perf-rt-' + n
|
||||
window.__MT__.ids.push({ sid, rid })
|
||||
const zone = ${zones} > 1 ? Math.floor((n - 1) / perZone) : 0
|
||||
const posInZone = ${zones} > 1 ? (n - 1) % perZone : n - 1
|
||||
const rank = posInZone * ${zones} + zone
|
||||
const isStreaming = rank < ${streaming}
|
||||
if (isStreaming) window.__MT__.streaming.push(rid)
|
||||
const leader = leaders[zone]
|
||||
if (leader) {
|
||||
hook.open(sid, 'center', 'session-tile:' + leader)
|
||||
} else if (${zones} === 1) {
|
||||
hook.open(sid, 'center')
|
||||
} else {
|
||||
leaders[zone] = sid
|
||||
if (zone === 0) hook.open(sid, 'right')
|
||||
else if (zone === 1) hook.open(sid, 'bottom', 'session-tile:' + leaders[0])
|
||||
else hook.open(sid, 'right', 'session-tile:' + leaders[zone - 2])
|
||||
}
|
||||
hook.patch(sid, { runtimeId: rid })
|
||||
hook.update(rid, () => state(sid, rid, isStreaming))
|
||||
}
|
||||
return 'ok'
|
||||
})()
|
||||
`
|
||||
|
||||
// Activate every tab once so keep-alive mounts the full stack (lazy mount:
|
||||
// a never-activated tab stays unmounted, which would understate the cost).
|
||||
const reveal = sid => `window.__HERMES_LAYOUT_TREE__.reveal(${JSON.stringify(`session-tile:${sid}`)})`
|
||||
|
||||
/** Page-side driver: grow every tile's streaming tail by `chunk` each
|
||||
* `intervalMs`, through the same write path the gateway flush uses.
|
||||
*
|
||||
* With `tools`, the stream is a working AGENT turn, not a monologue: every
|
||||
* 12th tick opens a live tool call on the streaming message (args, no
|
||||
* result — the running spinner), every 12th+6 completes it with a result
|
||||
* blob, and text keeps flowing between rounds. That exercises the tool-part
|
||||
* update path (find + replace inside the parts array) and the ToolCall
|
||||
* renderer's pending→complete transitions, which text-only streaming never
|
||||
* touches. */
|
||||
const drive = (chunk, intervalMs, totalTokens, tools) => `
|
||||
(() => {
|
||||
const hook = window.__HERMES_SESSION_TILES__
|
||||
let pushed = 0
|
||||
const tick = () => {
|
||||
for (const rid of window.__MT__.streaming) {
|
||||
hook.update(rid, prev => {
|
||||
if (!prev.streamId) return prev
|
||||
const messages = prev.messages.map(m => {
|
||||
if (m.id !== prev.streamId) return m
|
||||
const parts = m.parts.slice()
|
||||
if (${tools} && pushed % 12 === 0) {
|
||||
const args = { command: 'npm test -- --run suite-' + pushed, background: false }
|
||||
parts.push({ type: 'tool-call', toolCallId: rid + '-live-' + pushed, toolName: 'terminal',
|
||||
args, argsText: JSON.stringify(args) })
|
||||
} else if (${tools} && pushed % 12 === 6) {
|
||||
for (let p = parts.length - 1; p >= 0; p--) {
|
||||
const part = parts[p]
|
||||
if (part.type === 'tool-call' && part.result === undefined) {
|
||||
parts[p] = { ...part, result: JSON.stringify({ success: true,
|
||||
output: 'suite-' + pushed + ': 214 passed, 0 failed\\n'.repeat(12) }) }
|
||||
break
|
||||
}
|
||||
}
|
||||
parts.push({ type: 'text', text: '' })
|
||||
} else {
|
||||
const last = parts[parts.length - 1]
|
||||
if (last && last.type === 'text') {
|
||||
parts[parts.length - 1] = { type: 'text', text: last.text + ${JSON.stringify(chunk)} }
|
||||
} else {
|
||||
parts.push({ type: 'text', text: ${JSON.stringify(chunk)} })
|
||||
}
|
||||
}
|
||||
return { ...m, parts }
|
||||
})
|
||||
return { ...prev, messages }
|
||||
})
|
||||
}
|
||||
pushed += 1
|
||||
if (pushed < ${totalTokens}) window.__MT__.timer = setTimeout(tick, ${intervalMs})
|
||||
else window.__MT__.done = true
|
||||
}
|
||||
window.__MT__.timer = setTimeout(tick, ${intervalMs})
|
||||
return 'driving'
|
||||
})()
|
||||
`
|
||||
|
||||
const CLEANUP = `
|
||||
(() => {
|
||||
const hook = window.__HERMES_SESSION_TILES__
|
||||
if (window.__MT_DEAD__) {
|
||||
for (const rid of window.__MT_DEAD__) hook.drop?.(rid)
|
||||
window.__MT_DEAD__ = null
|
||||
}
|
||||
if (window.__MT__) {
|
||||
clearTimeout(window.__MT__.timer)
|
||||
for (const { sid, rid } of window.__MT__.ids) {
|
||||
// Settle through the real path so the in-flight journal entry clears.
|
||||
hook.update(rid, prev => ({ ...prev, busy: false, streamId: null }))
|
||||
hook.close(sid)
|
||||
}
|
||||
window.__MT__ = null
|
||||
}
|
||||
if (window.__MT_SAVED_SESSIONS__) {
|
||||
hook.seedSessions(window.__MT_SAVED_SESSIONS__)
|
||||
window.__MT_SAVED_SESSIONS__ = null
|
||||
}
|
||||
return 'cleaned'
|
||||
})()
|
||||
`
|
||||
|
||||
export default {
|
||||
name: 'multitab',
|
||||
tier: 'ci',
|
||||
description: 'N mounted session-tile tabs all streaming: frame pacing + longtasks.',
|
||||
async run(cdp, opts = {}) {
|
||||
const tiles = Number(opts.tiles ?? 5)
|
||||
const zones = Number(opts.zones ?? 1)
|
||||
const seedTurns = Number(opts.turns ?? 20)
|
||||
const seedSessions = Number(opts.sessions ?? 0)
|
||||
const streaming = Math.min(Number(opts.streaming ?? tiles), tiles)
|
||||
const dead = Number(opts.dead ?? 0)
|
||||
// --tools: seeded turns carry settled tool rounds and the live stream
|
||||
// opens/completes tool calls between text — an agent working, not talking.
|
||||
const tools = Boolean(opts.tools)
|
||||
const tokens = Number(opts.tokens ?? 240)
|
||||
// Matches STREAM_DELTA_FLUSH_MS — one publish per session per real flush.
|
||||
const intervalMs = Number(opts.intervalMs ?? 33)
|
||||
// --code: every tile grows ONE giant fenced code block with no settle
|
||||
// boundaries — what a coding agent streams. The block re-parses and
|
||||
// re-renders fully every flush (block memoization can't settle it), the
|
||||
// documented worst case and the "5 tabs all coding" crawl.
|
||||
const chunk = opts.code
|
||||
? ' const value = await resolve(ctx, { retry: true }) // step\n'
|
||||
: (opts.chunk ?? 'A streamed review sentence with **bold**, `code`, and ordinary prose.\n\n')
|
||||
const streamSeed = opts.code ? '```ts\n' : ''
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const ok = await cdp.eval(setup(tiles, seedTurns, streamSeed, zones, seedSessions, streaming, dead, tools))
|
||||
|
||||
if (ok !== 'ok') {
|
||||
throw new Error(`multitab setup failed (${ok}) — dev hooks missing? (needs a dev/probe renderer)`)
|
||||
}
|
||||
|
||||
// Mount every tab (keep-alive mounts on first activation), then settle.
|
||||
// Each reveal is timed to the next paint — with deep transcripts the
|
||||
// first mount is the "why does switching tabs hang" number.
|
||||
const revealMs = []
|
||||
|
||||
for (let n = 1; n <= tiles; n++) {
|
||||
const ms = Number(
|
||||
await cdp.eval(`
|
||||
new Promise(resolve => {
|
||||
const t0 = performance.now()
|
||||
${reveal(`perf-tile-${n}`)}
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve(performance.now() - t0)))
|
||||
})
|
||||
`)
|
||||
)
|
||||
|
||||
revealMs.push(ms)
|
||||
await sleep(350)
|
||||
}
|
||||
|
||||
// Front each zone's leader so the visible set is one transcript per zone
|
||||
// (the reveal loop above leaves each zone on its LAST tab).
|
||||
if (zones > 1) {
|
||||
const leaders = JSON.parse(await cdp.eval('JSON.stringify(window.__MT__.leaders)'))
|
||||
|
||||
for (const sid of leaders) {
|
||||
await cdp.eval(reveal(sid))
|
||||
await sleep(150)
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(1000)
|
||||
await cdp.eval(RECORDERS)
|
||||
await cdp.eval(drive(chunk, intervalMs, tokens, tools))
|
||||
await sleep(tokens * intervalMs + 1500)
|
||||
|
||||
const data = JSON.parse(await cdp.eval(COLLECT))
|
||||
await cdp.eval(CLEANUP)
|
||||
|
||||
// Drop the first 500ms (recorder install + settle).
|
||||
const frames = []
|
||||
let acc = 0
|
||||
|
||||
for (const f of data.frames) {
|
||||
acc += f
|
||||
|
||||
if (acc >= 500) {
|
||||
frames.push(f)
|
||||
}
|
||||
}
|
||||
|
||||
const ltDurations = data.longtasks.map(e => e.duration)
|
||||
const windowS = frames.reduce((a, b) => a + b, 0) / 1000
|
||||
// The felt numbers: sustained fps over the window, and the fps of the
|
||||
// worst 1-second slice (a 333ms frame IS "3fps" even if the average looks
|
||||
// fine). Worst slice = max summed frame time in any sliding 1s window.
|
||||
const avgFps = windowS ? frames.length / windowS : 0
|
||||
let worstFps = avgFps
|
||||
|
||||
for (let i = 0, j = 0, sum = 0; j < frames.length; j++) {
|
||||
sum += frames[j]
|
||||
|
||||
while (sum > 1000) {
|
||||
sum -= frames[i++]
|
||||
}
|
||||
|
||||
// Only a window that actually spans ~1s counts; short prefixes don't.
|
||||
if (sum >= 900) {
|
||||
worstFps = Math.min(worstFps, ((j - i + 1) / sum) * 1000)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
longtasks_n: data.longtasks.length,
|
||||
longtask_max_ms: Math.round((ltDurations.length ? Math.max(...ltDurations) : 0) * 10) / 10,
|
||||
frame_p95_ms: Math.round(percentile(frames, 0.95) * 10) / 10,
|
||||
frame_p99_ms: Math.round(percentile(frames, 0.99) * 10) / 10,
|
||||
slow_frames_33: frames.filter(f => f > 33).length,
|
||||
reveal_max_ms: Math.round(Math.max(...revealMs) * 10) / 10
|
||||
},
|
||||
detail: {
|
||||
tiles,
|
||||
zones,
|
||||
streaming,
|
||||
dead,
|
||||
sessions: seedSessions,
|
||||
tools,
|
||||
turns: seedTurns,
|
||||
code: Boolean(opts.code),
|
||||
windowS: Math.round(windowS * 10) / 10,
|
||||
avgFps: Math.round(avgFps * 10) / 10,
|
||||
worstSecondFps: Math.round(worstFps * 10) / 10,
|
||||
revealMs: revealMs.map(v => Math.round(v)),
|
||||
frameHistogram: frameHistogram(frames)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// Profile-switch latency. Subsumes measure-profile-switch. Backend tier: needs
|
||||
// a configured profile in the rail and a live backend. Report-only.
|
||||
//
|
||||
// node scripts/perf/run.mjs profile-switch --profile <name>
|
||||
|
||||
import { SELECTORS, sleep } from '../lib/cdp.mjs'
|
||||
|
||||
export default {
|
||||
name: 'profile-switch',
|
||||
tier: 'backend',
|
||||
description: 'Click a profile in the rail and wait for its sidebar to settle.',
|
||||
requiredOpts: ['profile'],
|
||||
async run(cdp, opts = {}) {
|
||||
const profile = opts.profile
|
||||
const settleTimeoutMs = Number(opts.settleTimeoutMs ?? 60000)
|
||||
|
||||
if (!profile) {
|
||||
throw new Error('profile-switch needs --profile <name>')
|
||||
}
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const t0 = await cdp.eval(`(() => {
|
||||
const rail = document.querySelector(${JSON.stringify(SELECTORS.profileRail)})
|
||||
if (!rail) return null
|
||||
const target = [...rail.querySelectorAll('button, [role="tab"]')].find(b =>
|
||||
((b.getAttribute('aria-label') || '') + ' ' + (b.title || '') + ' ' + (b.textContent || ''))
|
||||
.toLowerCase().includes(${JSON.stringify(String(profile).toLowerCase())}))
|
||||
if (!target) return null
|
||||
target.click()
|
||||
return performance.now()
|
||||
})()`)
|
||||
|
||||
if (t0 === null) {
|
||||
throw new Error(`profile "${profile}" not found in the rail`)
|
||||
}
|
||||
|
||||
const deadline = Date.now() + settleTimeoutMs
|
||||
let settledMs = null
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
await sleep(100)
|
||||
const s = await cdp.eval(`(() => {
|
||||
const label = [...document.querySelectorAll('div[aria-hidden]')].find(el => /waking up/i.test(el.textContent || ''))
|
||||
const overlayVisible = label ? Number(getComputedStyle(label).opacity) > 0.05 : false
|
||||
return { t: performance.now(), overlayVisible, rows: document.querySelectorAll(${JSON.stringify(SELECTORS.rowButton)}).length }
|
||||
})()`)
|
||||
|
||||
if (!s.overlayVisible && s.rows > 0) {
|
||||
settledMs = s.t - t0
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
metrics: { profile_switch_settled_ms: settledMs === null ? -1 : Math.round(settledMs) },
|
||||
detail: { profile, timedOut: settledMs === null }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
// Render churn during multi-tab streaming: WHAT re-rendered and WHY, and which
|
||||
// store published the update. Frame pacing (see `multitab`) tells you the cost;
|
||||
// this tells you the cause.
|
||||
//
|
||||
// Drives the same synthetic pipeline as `multitab` — publishSessionState per
|
||||
// session per flush via `__HERMES_SESSION_TILES__`, no backend, no credits —
|
||||
// then reads the dev-only counters installed by `src/debug/`:
|
||||
//
|
||||
// window.__RENDER_COUNTS__ — per-component renders, attributed to
|
||||
// props / hook state / parent-only ("wasted")
|
||||
// window.__ATOM_CHURN__ — per-store notifications, listener fan-out, and
|
||||
// notifications whose value was deep-equal to the
|
||||
// previous one ("wasted")
|
||||
//
|
||||
// The headline metric is `sidebar_renders`: how many times the sidebar tree
|
||||
// re-rendered while agents were typing in other tabs. It should be 0.
|
||||
//
|
||||
// node scripts/perf/run.mjs render-churn --spawn [--tiles 5] [--tokens 240]
|
||||
|
||||
import { sleep } from '../lib/cdp.mjs'
|
||||
|
||||
/** Components that make up the sidebar tree. A render of any of these while
|
||||
* a background tab streams is work the user cannot see. */
|
||||
const SIDEBAR_COMPONENTS = [
|
||||
'ChatSidebar',
|
||||
'SidebarSurface',
|
||||
'SessionRow',
|
||||
'SessionsSection',
|
||||
'CronJobsSection',
|
||||
'ProfileSwitcher',
|
||||
'VirtualSessionList',
|
||||
'WorkspaceGroup',
|
||||
'OverviewRow',
|
||||
'SessionStatusDot'
|
||||
]
|
||||
|
||||
/** Page-side setup: open `tiles` session tiles, seed each with a transcript.
|
||||
* Mirrors `multitab.mjs` so the two scenarios measure the same workload. */
|
||||
const setup = (tiles, seedTurns) => `
|
||||
(() => {
|
||||
const hook = window.__HERMES_SESSION_TILES__
|
||||
if (!hook) return 'no-hook'
|
||||
if (!window.__RENDER_COUNTS__) return 'no-render-counter'
|
||||
if (!window.__ATOM_CHURN__) return 'no-atom-churn'
|
||||
|
||||
const turn = (sid, i) => ([
|
||||
{ id: sid + '-u' + i, role: 'user', timestamp: Date.now(),
|
||||
parts: [{ type: 'text', text: 'Review question ' + i + ': does the diff handle the error path?' }] },
|
||||
{ id: sid + '-a' + i, role: 'assistant', timestamp: Date.now(), pending: false,
|
||||
parts: [{ type: 'text', text: '## Finding ' + i + '\\n\\nThe handler swallows the rejection.\\n\\n- The catch block drops the error.\\n- Retries are unbounded.\\n' }] }
|
||||
])
|
||||
|
||||
const state = (sid) => {
|
||||
const messages = []
|
||||
for (let i = 0; i < ${seedTurns}; i++) messages.push(...turn(sid, i))
|
||||
messages.push({ id: sid + '-stream', role: 'assistant', timestamp: Date.now(), pending: true,
|
||||
parts: [{ type: 'text', text: '' }] })
|
||||
return {
|
||||
storedSessionId: sid, messages, branch: '', cwd: '', model: '', provider: '',
|
||||
reasoningEffort: '', serviceTier: '', fast: false, yolo: false, personality: '',
|
||||
busy: true, awaitingResponse: false, streamId: sid + '-stream', sawAssistantPayload: true,
|
||||
pendingBranchGroup: null, interrupted: false, interimBoundaryPending: false,
|
||||
needsInput: false, turnStartedAt: Date.now(), usage: null
|
||||
}
|
||||
}
|
||||
|
||||
window.__RC__ = { ids: [], timer: null }
|
||||
for (let n = 1; n <= ${tiles}; n++) {
|
||||
const sid = 'churn-tile-' + n
|
||||
const rid = 'churn-rt-' + n
|
||||
window.__RC__.ids.push({ sid, rid })
|
||||
hook.open(sid, 'center')
|
||||
hook.patch(sid, { runtimeId: rid })
|
||||
hook.publish(rid, state(sid))
|
||||
}
|
||||
return 'ok'
|
||||
})()
|
||||
`
|
||||
|
||||
const reveal = sid => `window.__HERMES_LAYOUT_TREE__.reveal(${JSON.stringify(`session-tile:${sid}`)})`
|
||||
|
||||
/** Grow every tile's streaming tail by `chunk` each `intervalMs`, through the
|
||||
* same publish path the gateway's delta flush uses. */
|
||||
const drive = (chunk, intervalMs, totalTokens) => `
|
||||
(() => {
|
||||
const hook = window.__HERMES_SESSION_TILES__
|
||||
let pushed = 0
|
||||
const tick = () => {
|
||||
const states = hook.states()
|
||||
for (const { rid } of window.__RC__.ids) {
|
||||
const prev = states[rid]
|
||||
if (!prev) continue
|
||||
const messages = prev.messages.map(m => {
|
||||
if (m.id !== prev.streamId) return m
|
||||
const head = m.parts.slice(0, -1)
|
||||
const last = m.parts[m.parts.length - 1]
|
||||
return { ...m, parts: [...head, { type: 'text', text: last.text + ${JSON.stringify(chunk)} }] }
|
||||
})
|
||||
hook.publish(rid, { ...prev, messages })
|
||||
}
|
||||
pushed += 1
|
||||
if (pushed < ${totalTokens}) window.__RC__.timer = setTimeout(tick, ${intervalMs})
|
||||
else window.__RC__.done = true
|
||||
}
|
||||
window.__RC__.timer = setTimeout(tick, ${intervalMs})
|
||||
return 'driving'
|
||||
})()
|
||||
`
|
||||
|
||||
/** Wait until the renderer stops committing on its own, so the recording window
|
||||
* captures STREAMING cost and not whatever boot/hydration work happened to
|
||||
* still be in flight. Returns `quiet:N` once commits hold still for `quietMs`.
|
||||
*
|
||||
* If it returns `timeout:...` the app never went idle at all — with tiles
|
||||
* marked busy and NO driver running, that means something is ticking on its
|
||||
* own. The report of what rendered during the wait is attached so the culprit
|
||||
* is named rather than guessed at. */
|
||||
const quiesce = (quietMs, timeoutMs) => `
|
||||
(async () => {
|
||||
const rc = window.__RENDER_COUNTS__
|
||||
rc.start()
|
||||
const deadline = Date.now() + ${timeoutMs}
|
||||
const startedAt = Date.now()
|
||||
let last = -1
|
||||
let stableSince = Date.now()
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise(r => setTimeout(r, 100))
|
||||
const n = rc.commits()
|
||||
if (n !== last) { last = n; stableSince = Date.now(); continue }
|
||||
if (Date.now() - stableSince >= ${quietMs}) { rc.stop(); return 'quiet:' + n }
|
||||
}
|
||||
const idle = {
|
||||
commits: last,
|
||||
seconds: (Date.now() - startedAt) / 1000,
|
||||
top: rc.report(8),
|
||||
// Who OWNS the update? The component whose own hook state changed with
|
||||
// no changed props is the root of a churn cascade; everything under it
|
||||
// is collateral. Naming it is the difference between fixing the cause
|
||||
// and memoizing a symptom.
|
||||
owners: rc.report(200).filter(r => r.stateChanged > 0 && r.propsChanged === 0).slice(0, 8)
|
||||
}
|
||||
rc.stop()
|
||||
return 'timeout:' + JSON.stringify(idle)
|
||||
})()
|
||||
`
|
||||
|
||||
const START = `
|
||||
(() => {
|
||||
window.__RENDER_COUNTS__.start()
|
||||
window.__ATOM_CHURN__.start()
|
||||
return 'recording'
|
||||
})()
|
||||
`
|
||||
|
||||
const COLLECT = `
|
||||
(() => {
|
||||
window.__RENDER_COUNTS__.stop()
|
||||
window.__ATOM_CHURN__.stop()
|
||||
return JSON.stringify({
|
||||
commits: window.__RENDER_COUNTS__.commits(),
|
||||
renders: window.__RENDER_COUNTS__.report(200),
|
||||
atoms: window.__ATOM_CHURN__.report(200)
|
||||
})
|
||||
})()
|
||||
`
|
||||
|
||||
const CLEANUP = `
|
||||
(() => {
|
||||
if (window.__RC__) {
|
||||
clearTimeout(window.__RC__.timer)
|
||||
for (const { sid, rid } of window.__RC__.ids) {
|
||||
const states = window.__HERMES_SESSION_TILES__.states()
|
||||
window.__HERMES_SESSION_TILES__.publish(rid, { ...states[rid], busy: false, streamId: null })
|
||||
window.__HERMES_SESSION_TILES__.close(sid)
|
||||
}
|
||||
window.__RC__ = null
|
||||
}
|
||||
window.__RENDER_COUNTS__.clear()
|
||||
window.__ATOM_CHURN__.clear()
|
||||
return 'cleaned'
|
||||
})()
|
||||
`
|
||||
|
||||
export default {
|
||||
name: 'render-churn',
|
||||
tier: 'ci',
|
||||
description: 'N streaming tabs: per-component render attribution + store churn.',
|
||||
async run(cdp, opts = {}) {
|
||||
const tiles = Number(opts.tiles ?? 5)
|
||||
const seedTurns = Number(opts.turns ?? 20)
|
||||
const tokens = Number(opts.tokens ?? 240)
|
||||
// Matches STREAM_DELTA_FLUSH_MS — one publish per session per real flush.
|
||||
const intervalMs = Number(opts.intervalMs ?? 33)
|
||||
const chunk = opts.chunk ?? 'A streamed review sentence with **bold** and `code`.\n\n'
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const ok = await cdp.eval(setup(tiles, seedTurns))
|
||||
|
||||
if (ok !== 'ok') {
|
||||
throw new Error(
|
||||
`render-churn setup failed (${ok}) — needs a dev renderer with src/debug installed ` +
|
||||
'(the counters are aliased out of production builds unless VITE_PERF_PROBE=1).'
|
||||
)
|
||||
}
|
||||
|
||||
// Mount every tab (keep-alive mounts on first activation), then settle.
|
||||
for (let n = 1; n <= tiles; n++) {
|
||||
await cdp.eval(reveal(`churn-tile-${n}`))
|
||||
await sleep(350)
|
||||
}
|
||||
|
||||
// Let the app go quiet before recording, so boot/hydration commits that
|
||||
// happen to still be in flight don't land in the streaming window. This is
|
||||
// what makes runs comparable — a fixed sleep let 2-4x of hydration churn
|
||||
// leak in depending on machine load.
|
||||
const settle = await cdp.eval(quiesce(600, 15000))
|
||||
await cdp.eval(START)
|
||||
await cdp.eval(drive(chunk, intervalMs, tokens))
|
||||
await sleep(tokens * intervalMs + 1500)
|
||||
|
||||
const data = JSON.parse(await cdp.eval(COLLECT))
|
||||
await cdp.eval(CLEANUP)
|
||||
|
||||
const byName = new Map(data.renders.map(r => [r.name, r]))
|
||||
const sidebarRows = SIDEBAR_COMPONENTS.map(n => byName.get(n)).filter(Boolean)
|
||||
const sidebarRenders = sidebarRows.reduce((a, r) => a + r.renders, 0)
|
||||
const sidebarWasted = sidebarRows.reduce((a, r) => a + r.wasted, 0)
|
||||
const totalRenders = data.renders.reduce((a, r) => a + r.renders, 0)
|
||||
const totalWasted = data.renders.reduce((a, r) => a + r.wasted, 0)
|
||||
const atomWasted = data.atoms.reduce((a, r) => a + r.wasted, 0)
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
// The hypothesis, as a number: sidebar renders while background tabs
|
||||
// stream. Should be 0.
|
||||
sidebar_renders: sidebarRenders,
|
||||
sidebar_wasted: sidebarWasted,
|
||||
// Renders with no changed props and no changed hook state — pure
|
||||
// parent-driven work, across the whole tree.
|
||||
wasted_renders: totalWasted,
|
||||
total_renders: totalRenders,
|
||||
commits: data.commits,
|
||||
// Store notifications that published a value equal to the last one.
|
||||
wasted_notifies: atomWasted
|
||||
},
|
||||
detail: {
|
||||
tiles,
|
||||
tokens,
|
||||
// 'quiet:N' = the app went idle before recording (comparable run).
|
||||
// 'timeout:N' = it never did, so boot churn is mixed into the numbers.
|
||||
settle,
|
||||
sidebar: sidebarRows,
|
||||
topRenders: data.renders.slice(0, 15),
|
||||
topAtoms: data.atoms.slice(0, 15)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
// File-tree + terminal workspace stress. This is the regression scene for the
|
||||
// desktop symptom where opening the project tree/terminal made the whole page
|
||||
// hitch while chat and PTY output continued.
|
||||
//
|
||||
// It mounts a real project tree, one PTY plus multiple persistent xterm tabs,
|
||||
// streams chat and terminal output together, mutates Git decoration state, and
|
||||
// drags the terminal split. The debug probe records the specific work we care
|
||||
// about rather than inferring it from CPU alone:
|
||||
// - fixed-overlay measurements
|
||||
// - active/hidden xterm fits
|
||||
// - ProjectTree + per-path row renders
|
||||
// - frame pacing / slow frames
|
||||
//
|
||||
// npm run perf -- right-pane --spawn --prod --runs 3
|
||||
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { sleep } from '../lib/cdp.mjs'
|
||||
import { frameHistogram, percentile } from '../lib/stats.mjs'
|
||||
|
||||
const DEFAULT_CWD = resolve(dirname(fileURLToPath(import.meta.url)), '../../..')
|
||||
|
||||
const RECORDERS = `
|
||||
(() => {
|
||||
window.__RP_FRAME_GEN__ = (window.__RP_FRAME_GEN__ || 0) + 1
|
||||
const generation = window.__RP_FRAME_GEN__
|
||||
window.__RP_FRAMES__ = { times: [], stop: false }
|
||||
let last = performance.now()
|
||||
const tick = () => {
|
||||
if (window.__RP_FRAME_GEN__ !== generation || window.__RP_FRAMES__.stop) return
|
||||
const now = performance.now()
|
||||
window.__RP_FRAMES__.times.push(now - last)
|
||||
last = now
|
||||
requestAnimationFrame(tick)
|
||||
}
|
||||
requestAnimationFrame(tick)
|
||||
|
||||
window.__RP_LONG__ = { entries: [], stop: false }
|
||||
try {
|
||||
const observer = new PerformanceObserver(list => {
|
||||
if (window.__RP_LONG__.stop) return
|
||||
for (const entry of list.getEntries()) {
|
||||
window.__RP_LONG__.entries.push({ duration: entry.duration, startTime: entry.startTime })
|
||||
}
|
||||
})
|
||||
observer.observe({ entryTypes: ['longtask'] })
|
||||
window.__RP_LONG__.observer = observer
|
||||
} catch {}
|
||||
return 'armed'
|
||||
})()
|
||||
`
|
||||
|
||||
const COLLECT_RECORDERS = `
|
||||
(() => {
|
||||
window.__RP_FRAMES__.stop = true
|
||||
window.__RP_LONG__.stop = true
|
||||
try { window.__RP_LONG__.observer && window.__RP_LONG__.observer.disconnect() } catch {}
|
||||
return JSON.stringify({ frames: window.__RP_FRAMES__.times, longtasks: window.__RP_LONG__.entries })
|
||||
})()
|
||||
`
|
||||
|
||||
const START_COUNTERS = `window.__RIGHT_PANE_PERF__.start(); 'recording'`
|
||||
const SNAPSHOT_COUNTERS = `
|
||||
(() => {
|
||||
window.__RIGHT_PANE_PERF__.stop()
|
||||
return JSON.stringify(window.__RIGHT_PANE_PERF__.snapshot())
|
||||
})()
|
||||
`
|
||||
|
||||
const DRAG_TERMINAL_SPLIT = `
|
||||
(async () => {
|
||||
const slot = document.querySelector('[data-terminal-slot]')
|
||||
const overlay = document.querySelector('[data-persistent-terminal]')
|
||||
if (!slot || !overlay) return JSON.stringify({ target: 'none', drift: -1, moved: 0 })
|
||||
|
||||
const slotBox = slot.getBoundingClientRect()
|
||||
const candidates = [...document.querySelectorAll('[role="separator"]')]
|
||||
.map(element => ({ element, box: element.getBoundingClientRect() }))
|
||||
.filter(item => item.box.width > item.box.height * 3)
|
||||
.sort((a, b) =>
|
||||
Math.abs((a.box.top + a.box.bottom) / 2 - slotBox.top) -
|
||||
Math.abs((b.box.top + b.box.bottom) / 2 - slotBox.top)
|
||||
)
|
||||
const target = candidates[0]
|
||||
if (!target) return JSON.stringify({ target: 'none', drift: -1, moved: 0 })
|
||||
|
||||
const x = target.box.left + target.box.width / 2
|
||||
const y0 = target.box.top + target.box.height / 2
|
||||
let y = y0
|
||||
const pointer = {
|
||||
bubbles: true, cancelable: true, pointerId: 91, pointerType: 'mouse',
|
||||
isPrimary: true, button: 0, buttons: 1
|
||||
}
|
||||
target.element.dispatchEvent(new PointerEvent('pointerdown', { ...pointer, clientX: x, clientY: y }))
|
||||
|
||||
for (let i = 0; i < 24; i += 1) {
|
||||
y -= 1
|
||||
window.dispatchEvent(new PointerEvent('pointermove', { ...pointer, clientX: x, clientY: y }))
|
||||
await new Promise(resolve => requestAnimationFrame(resolve))
|
||||
}
|
||||
for (let i = 0; i < 24; i += 1) {
|
||||
y += 1
|
||||
window.dispatchEvent(new PointerEvent('pointermove', { ...pointer, clientX: x, clientY: y }))
|
||||
await new Promise(resolve => requestAnimationFrame(resolve))
|
||||
}
|
||||
window.dispatchEvent(new PointerEvent('pointerup', { ...pointer, buttons: 0, clientX: x, clientY: y }))
|
||||
// Track-size transitions continue briefly after pointerup. Wait through
|
||||
// that animation, then give the overlay its normal two-frame calibration.
|
||||
await new Promise(resolve => setTimeout(resolve, 350))
|
||||
await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))
|
||||
|
||||
const a = slot.getBoundingClientRect()
|
||||
const b = overlay.getBoundingClientRect()
|
||||
const drift = Math.max(
|
||||
Math.abs(a.top - b.top),
|
||||
Math.abs(a.left - b.left),
|
||||
Math.abs(a.width - b.width),
|
||||
Math.abs(a.height - b.height)
|
||||
)
|
||||
return JSON.stringify({ target: 'horizontal-separator', drift, moved: 24 })
|
||||
})()
|
||||
`
|
||||
|
||||
async function waitFor(cdp, expression, label, timeoutMs = 20000) {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
if (await cdp.eval(expression)) {
|
||||
return
|
||||
}
|
||||
|
||||
await sleep(100)
|
||||
}
|
||||
|
||||
throw new Error(`right-pane timed out waiting for ${label}`)
|
||||
}
|
||||
|
||||
const trimWarmup = (frames, warmupMs = 300) => {
|
||||
const kept = []
|
||||
let elapsed = 0
|
||||
|
||||
for (const frame of frames) {
|
||||
elapsed += frame
|
||||
|
||||
if (elapsed >= warmupMs) {
|
||||
kept.push(frame)
|
||||
}
|
||||
}
|
||||
|
||||
return kept
|
||||
}
|
||||
|
||||
export default {
|
||||
name: 'right-pane',
|
||||
tier: 'report',
|
||||
description: 'Project tree + persistent terminal tabs under chat/terminal output and split dragging.',
|
||||
async run(cdp, opts = {}) {
|
||||
const cwd = resolve(String(opts.cwd ?? DEFAULT_CWD))
|
||||
const terminalCount = Math.max(2, Number(opts.terminals ?? 3))
|
||||
const tokens = Number(opts.tokens ?? 90)
|
||||
const outputChunks = Number(opts.outputChunks ?? 160)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const ready = await cdp.eval(
|
||||
`!!(window.__PERF_DRIVE__?.rightPaneSetup && window.__RIGHT_PANE_PERF__ && window.__HERMES_LAYOUT_TREE__)`
|
||||
)
|
||||
|
||||
if (!ready) {
|
||||
throw new Error('right-pane needs a dev renderer or a production build with VITE_PERF_PROBE=1.')
|
||||
}
|
||||
|
||||
let setup
|
||||
|
||||
try {
|
||||
setup = await cdp.eval(
|
||||
`window.__PERF_DRIVE__.rightPaneSetup(${JSON.stringify({ cwd, terminals: terminalCount })})`
|
||||
)
|
||||
await cdp.eval(`window.__HERMES_LAYOUT_TREE__.reveal('files'); window.__HERMES_LAYOUT_TREE__.reveal('terminal')`)
|
||||
await waitFor(cdp, `!!document.querySelector('[data-project-tree]')`, 'project tree')
|
||||
await waitFor(
|
||||
cdp,
|
||||
`!!document.querySelector('[data-terminal-slot]') && !!document.querySelector('[data-persistent-terminal]')`,
|
||||
'persistent terminal'
|
||||
)
|
||||
await waitFor(
|
||||
cdp,
|
||||
`document.querySelectorAll('[data-terminal] .xterm').length >= ${terminalCount}`,
|
||||
`${terminalCount} mounted xterms`,
|
||||
30000
|
||||
)
|
||||
await sleep(1200)
|
||||
|
||||
// Activate every keep-alive tab once, then return to the output tab.
|
||||
// Each activation should restore exactly one fit; inactive tabs must stay
|
||||
// at zero even while another tab resizes or writes output.
|
||||
await cdp.eval(START_COUNTERS)
|
||||
|
||||
for (const id of setup.terminalIds) {
|
||||
await cdp.eval(`window.__PERF_DRIVE__.rightPaneSelect(${JSON.stringify(id)})`)
|
||||
await sleep(180)
|
||||
}
|
||||
|
||||
const activation = JSON.parse(await cdp.eval(SNAPSHOT_COUNTERS))
|
||||
|
||||
await cdp.eval(RECORDERS)
|
||||
|
||||
// Chat DOM churn is deliberately measured in its own counter window:
|
||||
// terminal positioning should receive no wakeups from transcript changes.
|
||||
await cdp.eval(START_COUNTERS)
|
||||
await cdp.eval(
|
||||
`window.__PERF_DRIVE__.stream({
|
||||
chunk: 'Right pane streaming sentence with **bold** and \`code\`.\\n\\n',
|
||||
intervalMs: 16,
|
||||
totalTokens: ${tokens},
|
||||
flushMinMs: 33
|
||||
})`
|
||||
)
|
||||
await cdp.eval(`
|
||||
(() => {
|
||||
let n = 0
|
||||
window.__RP_OUTPUT_TIMER__ = setInterval(() => {
|
||||
window.__PERF_DRIVE__.rightPaneWrite(
|
||||
${JSON.stringify(setup.procId)},
|
||||
'terminal output line ' + n + ' ........................................\\r\\n'
|
||||
)
|
||||
n += 1
|
||||
if (n >= ${outputChunks}) clearInterval(window.__RP_OUTPUT_TIMER__)
|
||||
}, 16)
|
||||
return 'writing'
|
||||
})()
|
||||
`)
|
||||
await sleep(Math.max(tokens, outputChunks) * 16 + 900)
|
||||
const stream = JSON.parse(await cdp.eval(SNAPSHOT_COUNTERS))
|
||||
|
||||
// An unrelated Git status publication should render neither the tree root
|
||||
// nor any visible row. A status for one visible path should touch only it.
|
||||
await cdp.eval(START_COUNTERS)
|
||||
await cdp.eval(`window.__PERF_DRIVE__.rightPaneGit('__right_pane_unrelated__.txt', 'modified')`)
|
||||
await sleep(250)
|
||||
const unrelatedGit = JSON.parse(await cdp.eval(SNAPSHOT_COUNTERS))
|
||||
|
||||
const visiblePath = await cdp.eval(
|
||||
`document.querySelector('[data-project-tree] [title]')?.getAttribute('title') || ''`
|
||||
)
|
||||
let affectedGit = { counts: { 'project-tree-render': 0, 'project-tree-row-render': 0 }, rows: {} }
|
||||
|
||||
if (visiblePath) {
|
||||
const relative = String(visiblePath).startsWith(`${cwd}/`)
|
||||
? String(visiblePath).slice(cwd.length + 1)
|
||||
: String(visiblePath)
|
||||
await cdp.eval(START_COUNTERS)
|
||||
await cdp.eval(`window.__PERF_DRIVE__.rightPaneGit(${JSON.stringify(relative)}, 'modified')`)
|
||||
await sleep(250)
|
||||
affectedGit = JSON.parse(await cdp.eval(SNAPSHOT_COUNTERS))
|
||||
}
|
||||
|
||||
await cdp.eval(START_COUNTERS)
|
||||
const drag = JSON.parse(await cdp.eval(DRAG_TERMINAL_SPLIT))
|
||||
const dragCounters = JSON.parse(await cdp.eval(SNAPSHOT_COUNTERS))
|
||||
const recorded = JSON.parse(await cdp.eval(COLLECT_RECORDERS))
|
||||
const frames = trimWarmup(recorded.frames)
|
||||
const longtasks = recorded.longtasks.map(entry => entry.duration)
|
||||
const streamCounts = stream.counts
|
||||
const activationCounts = activation.counts
|
||||
const unrelatedCounts = unrelatedGit.counts
|
||||
const affectedRows = Object.values(affectedGit.rows).reduce((sum, count) => sum + count, 0)
|
||||
const affectedPaths = Object.keys(affectedGit.rows).length
|
||||
|
||||
if (drag.target === 'none') {
|
||||
throw new Error('right-pane found no horizontal terminal split separator.')
|
||||
}
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
chat_terminal_measures: streamCounts['terminal-measure'],
|
||||
hidden_terminal_fits: activationCounts['terminal-fit-hidden'] + streamCounts['terminal-fit-hidden'],
|
||||
activation_fit_mismatch: Math.abs(activationCounts['terminal-fit-active'] - setup.terminalIds.length),
|
||||
unrelated_tree_renders: unrelatedCounts['project-tree-render'],
|
||||
unrelated_row_renders: unrelatedCounts['project-tree-row-render'],
|
||||
affected_tree_renders: affectedGit.counts['project-tree-render'],
|
||||
affected_row_path_excess: Math.max(0, affectedPaths - 1),
|
||||
terminal_drift_px: Math.round(drag.drift * 10) / 10,
|
||||
frame_p95_ms: Math.round(percentile(frames, 0.95) * 10) / 10,
|
||||
frame_p99_ms: Math.round(percentile(frames, 0.99) * 10) / 10,
|
||||
slow_frames_33: frames.filter(frame => frame > 33).length,
|
||||
longtask_max_ms: Math.round((longtasks.length ? Math.max(...longtasks) : 0) * 10) / 10
|
||||
},
|
||||
detail: {
|
||||
cwd,
|
||||
terminals: setup.terminalIds.length,
|
||||
activation,
|
||||
stream,
|
||||
unrelatedGit,
|
||||
affectedGit,
|
||||
affectedRows,
|
||||
drag,
|
||||
dragCounters,
|
||||
frameHistogram: frameHistogram(frames),
|
||||
frames: frames.length
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await cdp.eval(`
|
||||
(() => {
|
||||
clearInterval(window.__RP_OUTPUT_TIMER__)
|
||||
window.__RIGHT_PANE_PERF__?.stop()
|
||||
window.__PERF_DRIVE__?.reset()
|
||||
return 'cleaned'
|
||||
})()
|
||||
`)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
// Visual stability of a session LOAD. Sibling to `submit` (which measures the
|
||||
// jump on Enter); this one measures the jump on opening a session — the
|
||||
// prepend/settle path, not the append path.
|
||||
//
|
||||
// Clicks sidebar rows and tracks the bottom-most turn's on-screen top every
|
||||
// frame. A clean load never moves it after first paint; a janky one strands it
|
||||
// thousands of px away while the render-budget backfill and stick-to-bottom
|
||||
// argue. Backend tier: needs real stored sessions in the sidebar.
|
||||
//
|
||||
// node scripts/perf/run.mjs session-load --rows 2,5,8 --rounds 2
|
||||
|
||||
import { SELECTORS, sleep } from '../lib/cdp.mjs'
|
||||
import { summarize } from '../lib/stats.mjs'
|
||||
|
||||
// Below this a shift is sub-perceptual (sub-pixel rounding, a settling caret).
|
||||
const SHIFT_PX = 4
|
||||
|
||||
const ARM = `
|
||||
(() => {
|
||||
const samples = []
|
||||
const t0 = performance.now()
|
||||
let running = true
|
||||
|
||||
const tick = () => {
|
||||
if (!running) return
|
||||
const v = document.querySelector(${JSON.stringify(SELECTORS.threadViewport)})
|
||||
|
||||
if (v) {
|
||||
const turns = v.querySelectorAll(
|
||||
${JSON.stringify(SELECTORS.turnPair)} + ',' + ${JSON.stringify(SELECTORS.assistantMessage)}
|
||||
)
|
||||
const last = turns[turns.length - 1]
|
||||
const rect = last && last.getBoundingClientRect()
|
||||
samples.push({
|
||||
st: Math.round(v.scrollTop),
|
||||
sh: v.scrollHeight,
|
||||
ch: v.clientHeight,
|
||||
bottomTop: rect ? Math.round(rect.top - v.getBoundingClientRect().top) : null,
|
||||
turns: turns.length,
|
||||
t: Math.round(performance.now() - t0)
|
||||
})
|
||||
}
|
||||
|
||||
requestAnimationFrame(tick)
|
||||
}
|
||||
|
||||
requestAnimationFrame(tick)
|
||||
window.__SL = { samples, stop() { running = false } }
|
||||
})()
|
||||
`
|
||||
|
||||
const CLICK = index => `
|
||||
(() => {
|
||||
const rows = [...document.querySelectorAll(${JSON.stringify(SELECTORS.rowButton)})].filter(el => el.offsetParent)
|
||||
const row = rows[${index}]
|
||||
if (!row) return null
|
||||
row.click()
|
||||
return (row.textContent ?? '').slice(0, 34)
|
||||
})()
|
||||
`
|
||||
|
||||
/** Total/max on-screen movement of the bottom turn after it first paints. */
|
||||
function measureLoad(samples) {
|
||||
const painted = samples.findIndex(s => s.turns > 0)
|
||||
const after = painted === -1 ? [] : samples.slice(painted)
|
||||
const load = { maxShiftPx: 0, offBottomFrames: 0, settledMs: 0, shiftedPx: 0, shifts: 0 }
|
||||
let previous = null
|
||||
|
||||
for (const sample of after) {
|
||||
if (sample.sh - (sample.st + sample.ch) > 2) {
|
||||
load.offBottomFrames += 1
|
||||
}
|
||||
|
||||
if (previous?.bottomTop != null && sample.bottomTop != null) {
|
||||
const delta = Math.abs(sample.bottomTop - previous.bottomTop)
|
||||
|
||||
if (delta > SHIFT_PX) {
|
||||
load.maxShiftPx = Math.max(load.maxShiftPx, delta)
|
||||
load.settledMs = sample.t
|
||||
load.shiftedPx += delta
|
||||
load.shifts += 1
|
||||
}
|
||||
}
|
||||
|
||||
previous = sample
|
||||
}
|
||||
|
||||
return load
|
||||
}
|
||||
|
||||
export default {
|
||||
name: 'session-load',
|
||||
tier: 'backend',
|
||||
description: 'Visual stability of opening a session: how far the transcript moves after first paint.',
|
||||
async run(cdp, opts = {}) {
|
||||
const rows = String(opts.rows ?? '2,5,8').split(',').map(Number)
|
||||
const rounds = Number(opts.rounds ?? 2)
|
||||
const watchMs = Number(opts.watchMs ?? 4500)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const loads = []
|
||||
|
||||
for (let round = 0; round < rounds; round++) {
|
||||
for (const index of rows) {
|
||||
await cdp.eval(ARM)
|
||||
|
||||
if (!(await cdp.eval(CLICK(index)))) {
|
||||
continue
|
||||
}
|
||||
|
||||
await sleep(watchMs)
|
||||
const { samples } = await cdp.eval('(() => { window.__SL.stop(); return window.__SL })()')
|
||||
loads.push(measureLoad(samples))
|
||||
}
|
||||
|
||||
await sleep(500)
|
||||
}
|
||||
|
||||
if (!loads.length) {
|
||||
throw new Error('session-load found no sidebar rows to click')
|
||||
}
|
||||
|
||||
const total = key => loads.reduce((sum, load) => sum + load[key], 0)
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
load_max_shift_px: Math.max(...loads.map(load => load.maxShiftPx)),
|
||||
load_off_bottom_frames: Math.round(total('offBottomFrames') / loads.length),
|
||||
load_settled_p95_ms: summarize(loads.map(load => load.settledMs)).p95,
|
||||
load_shifted_px: Math.round(total('shiftedPx') / loads.length)
|
||||
},
|
||||
detail: { loads: loads.length, shiftsPerLoad: total('shifts') / loads.length }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// Session-switch latency. Subsumes profile-session-switch. Backend tier: needs
|
||||
// two real stored session ids and a live backend. Report-only.
|
||||
//
|
||||
// node scripts/perf/run.mjs session-switch --a <sidA> --b <sidB> [--rounds 2]
|
||||
|
||||
import { SELECTORS, sleep } from '../lib/cdp.mjs'
|
||||
import { summarize } from '../lib/stats.mjs'
|
||||
|
||||
export default {
|
||||
name: 'session-switch',
|
||||
tier: 'backend',
|
||||
description: 'Route to a session and wait for first-paint + settle of its transcript.',
|
||||
requiredOpts: ['a', 'b'],
|
||||
async run(cdp, opts = {}) {
|
||||
const { a, b } = opts
|
||||
const rounds = Number(opts.rounds ?? 2)
|
||||
const settleTimeoutMs = Number(opts.settleTimeoutMs ?? 30000)
|
||||
|
||||
if (!a || !b) {
|
||||
throw new Error('session-switch needs --a <sessionId> --b <sessionId>')
|
||||
}
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const switchTo = async sid => {
|
||||
const t0 = await cdp.eval(`(() => { location.hash = '#/' + ${JSON.stringify(sid)}; return performance.now() })()`)
|
||||
const deadline = Date.now() + settleTimeoutMs
|
||||
let firstPaint = null
|
||||
let stable = 0
|
||||
let lastCount = -1
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
await sleep(50)
|
||||
const s = await cdp.eval(`({
|
||||
t: performance.now(),
|
||||
route: location.hash,
|
||||
msgs: document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length
|
||||
})`)
|
||||
|
||||
if (!String(s.route).includes(sid)) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (s.msgs > 0 && firstPaint === null) {
|
||||
firstPaint = s.t - t0
|
||||
}
|
||||
|
||||
stable = s.msgs === lastCount && s.msgs > 0 ? stable + 1 : 0
|
||||
lastCount = s.msgs
|
||||
|
||||
if (stable >= 3) {
|
||||
return { firstPaint, settled: s.t - t0 }
|
||||
}
|
||||
}
|
||||
|
||||
return { firstPaint, settled: null }
|
||||
}
|
||||
|
||||
const firstPaints = []
|
||||
const settles = []
|
||||
|
||||
for (let round = 0; round < rounds; round++) {
|
||||
for (const sid of [a, b]) {
|
||||
const r = await switchTo(sid)
|
||||
|
||||
if (typeof r.firstPaint === 'number') firstPaints.push(r.firstPaint)
|
||||
if (typeof r.settled === 'number') settles.push(r.settled)
|
||||
await sleep(800)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
switch_first_paint_p95_ms: summarize(firstPaints).p95,
|
||||
switch_settled_p95_ms: summarize(settles).p95
|
||||
},
|
||||
detail: { rounds, firstPaint: summarize(firstPaints), settled: summarize(settles) }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Streaming into an ALREADY-LONG transcript. Same measurement as `stream`, but
|
||||
// the history is mounted and allowed to settle before the recorders start, so
|
||||
// what it captures is the per-delta cost that scales with transcript length —
|
||||
// the regression reported in #69120.
|
||||
//
|
||||
// Report-only (tier: manual): the number depends on how much history the host
|
||||
// can mount, so it is not gated against the committed baseline.
|
||||
|
||||
import stream from './stream.mjs'
|
||||
|
||||
export default {
|
||||
name: 'stream-history',
|
||||
tier: 'manual',
|
||||
description: 'Streaming cost with a long settled transcript already mounted.',
|
||||
run(cdp, opts = {}) {
|
||||
return stream.run(cdp, { ...opts, historyTurns: Number(opts.historyTurns ?? 200) })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
// Streaming render cost. Subsumes measure-synthetic-stream, profile-synth-stream,
|
||||
// profile-long-stream (synthetic) and measure-real-stream / profile-real-stream
|
||||
// (via --real). CPU profiling is provided by the runner's --cpuprofile flag.
|
||||
//
|
||||
// Metrics (lower is better): longtask count + max, frame p95/p99, slow-frame
|
||||
// count, inter-mutation p95. These are what "is streaming smooth?" reduces to.
|
||||
|
||||
import { SELECTORS, sleep, typeIntoComposer } from '../lib/cdp.mjs'
|
||||
import { frameHistogram, percentile } from '../lib/stats.mjs'
|
||||
|
||||
const RECORDERS = `
|
||||
(() => {
|
||||
// Generation guard: a prior run's rAF loop re-reads window.__FT__ each frame,
|
||||
// so simply reassigning it would leave the old loop running and pushing into
|
||||
// the new array (overlapping recorders inflate frame intervals on run 2+).
|
||||
// Bumping the generation makes every stale loop exit on its next tick.
|
||||
window.__FT_GEN__ = (window.__FT_GEN__ || 0) + 1
|
||||
const ftGen = window.__FT_GEN__
|
||||
window.__FT__ = { times: [], stop: false }
|
||||
let last = performance.now()
|
||||
const tick = () => {
|
||||
if (window.__FT_GEN__ !== ftGen || window.__FT__.stop) return
|
||||
const now = performance.now()
|
||||
window.__FT__.times.push(now - last)
|
||||
last = now
|
||||
requestAnimationFrame(tick)
|
||||
}
|
||||
requestAnimationFrame(tick)
|
||||
|
||||
window.__LT__ = { entries: [], stop: false }
|
||||
try {
|
||||
const po = new PerformanceObserver((list) => {
|
||||
if (window.__LT__.stop) return
|
||||
for (const e of list.getEntries()) window.__LT__.entries.push({ duration: e.duration, startTime: e.startTime })
|
||||
})
|
||||
po.observe({ entryTypes: ['longtask'] })
|
||||
window.__LT__.po = po
|
||||
} catch {}
|
||||
|
||||
window.__MO__ = { mutations: [], stop: false, current: null }
|
||||
window.__MO__.arm = () => {
|
||||
const all = document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)})
|
||||
const last = all[all.length - 1]
|
||||
if (!last || last === window.__MO__.current) return
|
||||
window.__MO__.current = last
|
||||
window.__MO__.obs && window.__MO__.obs.disconnect()
|
||||
const obs = new MutationObserver(() => {
|
||||
if (window.__MO__.stop) return
|
||||
window.__MO__.mutations.push({ t: performance.now(), len: last.textContent.length })
|
||||
})
|
||||
obs.observe(last, { childList: true, subtree: true, characterData: true })
|
||||
window.__MO__.obs = obs
|
||||
}
|
||||
return 'armed'
|
||||
})()
|
||||
`
|
||||
|
||||
const COLLECT = `
|
||||
(() => {
|
||||
window.__FT__.stop = true
|
||||
window.__LT__.stop = true
|
||||
window.__MO__.stop = true
|
||||
try { window.__LT__.po && window.__LT__.po.disconnect() } catch {}
|
||||
try { window.__MO__.obs && window.__MO__.obs.disconnect() } catch {}
|
||||
return JSON.stringify({
|
||||
frames: window.__FT__.times,
|
||||
longtasks: window.__LT__.entries,
|
||||
mutations: window.__MO__.mutations
|
||||
})
|
||||
})()
|
||||
`
|
||||
|
||||
function analyze(data, warmupMs, extra = {}) {
|
||||
// Drop warm-up frames (recorder installs before the stream starts).
|
||||
const frames = []
|
||||
let acc = 0
|
||||
|
||||
for (const f of data.frames) {
|
||||
acc += f
|
||||
|
||||
if (acc >= warmupMs) {
|
||||
frames.push(f)
|
||||
}
|
||||
}
|
||||
|
||||
const interMut = []
|
||||
|
||||
for (let i = 1; i < data.mutations.length; i++) {
|
||||
interMut.push(data.mutations[i].t - data.mutations[i - 1].t)
|
||||
}
|
||||
|
||||
const ltDurations = data.longtasks.map(e => e.duration)
|
||||
const windowS = frames.reduce((a, b) => a + b, 0) / 1000
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
longtasks_n: data.longtasks.length,
|
||||
longtask_max_ms: Math.round((ltDurations.length ? Math.max(...ltDurations) : 0) * 10) / 10,
|
||||
frame_p95_ms: Math.round(percentile(frames, 0.95) * 10) / 10,
|
||||
frame_p99_ms: Math.round(percentile(frames, 0.99) * 10) / 10,
|
||||
slow_frames_33: frames.filter(f => f > 33).length,
|
||||
intermut_p95_ms: Math.round(percentile(interMut, 0.95) * 10) / 10
|
||||
},
|
||||
detail: {
|
||||
...extra,
|
||||
windowS: Math.round(windowS * 10) / 10,
|
||||
avgFps: windowS ? Math.round((frames.length / windowS) * 10) / 10 : 0,
|
||||
frameHistogram: frameHistogram(frames),
|
||||
mutations: data.mutations.length,
|
||||
finalLen: data.mutations.at(-1)?.len ?? 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
name: 'stream',
|
||||
tier: 'ci',
|
||||
description: 'Assistant-message streaming: longtasks, frame pacing, mutation cadence.',
|
||||
async run(cdp, opts = {}) {
|
||||
const tokens = Number(opts.tokens ?? 400)
|
||||
const intervalMs = Number(opts.intervalMs ?? 16)
|
||||
const flushMinMs = Number(opts.flushMinMs ?? 33)
|
||||
// Realistic default: a short markdown paragraph ending in a blank line, so
|
||||
// blocks SETTLE as they stream — exactly how real LLM output behaves, and
|
||||
// what block-memoization is designed for (only the growing tail re-renders).
|
||||
// A chunk with NO paragraph break (e.g. `--chunk 'word '`) instead grows one
|
||||
// ever-larger block that re-renders fully every flush — a useful worst-case
|
||||
// stress, but not the typical number. No raw autolink (avoids DNS/link-embed
|
||||
// noise unrelated to render cost).
|
||||
const chunk = opts.chunk ?? 'A streamed sentence with **bold**, `code`, and ordinary prose like a normal reply.\n\n'
|
||||
const real = Boolean(opts.real)
|
||||
const historyTurns = Number(opts.historyTurns ?? 0)
|
||||
const historySettleMs = Number(opts.historySettleMs ?? 1500)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
// Mount the settled history BEFORE the recorders start, so the measurement
|
||||
// window contains only streaming work — not the one-off mount cost.
|
||||
if (historyTurns > 0) {
|
||||
if (real) {
|
||||
throw new Error('--historyTurns is only supported by the synthetic stream path')
|
||||
}
|
||||
|
||||
await cdp.eval(`window.__PERF_DRIVE__.loadTranscript(${historyTurns})`)
|
||||
await sleep(historySettleMs)
|
||||
|
||||
const mounted = Number(await cdp.eval('window.__PERF_DRIVE__.snapshotMsgs()'))
|
||||
const expected = historyTurns * 2
|
||||
|
||||
if (mounted !== expected) {
|
||||
throw new Error(`expected ${expected} preloaded history messages, got ${mounted}`)
|
||||
}
|
||||
}
|
||||
|
||||
await cdp.eval(RECORDERS)
|
||||
|
||||
if (real) {
|
||||
// Backend path: fire a real prompt and wait for the stream to appear.
|
||||
const baseCount = await cdp.eval(
|
||||
`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`
|
||||
)
|
||||
await typeIntoComposer(cdp, opts.prompt ?? 'count from 1 to 80, one number per line', { cps: 40 })
|
||||
await cdp.eval(`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
||||
el && el.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', bubbles: true, cancelable: true }))
|
||||
})()`)
|
||||
|
||||
const deadline = Date.now() + Number(opts.timeoutMs ?? 60000)
|
||||
let started = false
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
await sleep(50)
|
||||
const n = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`)
|
||||
|
||||
if (n > baseCount) {
|
||||
started = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (!started) {
|
||||
throw new Error('real stream never started (no LLM credit / backend?)')
|
||||
}
|
||||
|
||||
await cdp.eval('window.__MO__.arm()')
|
||||
// Let it run to completion or timeout.
|
||||
const runDeadline = Date.now() + Number(opts.runMs ?? 30000)
|
||||
|
||||
while (Date.now() < runDeadline) {
|
||||
await sleep(250)
|
||||
const busy = await cdp.eval(`!!document.querySelector('[data-status="running"], [data-busy="true"]')`)
|
||||
|
||||
if (!busy) {
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Synthetic path: drive $messages directly. No LLM, no credits.
|
||||
await cdp.eval(
|
||||
`window.__PERF_DRIVE__.stream({ chunk: ${JSON.stringify(chunk)}, intervalMs: ${intervalMs}, totalTokens: ${tokens}, flushMinMs: ${flushMinMs} })`
|
||||
)
|
||||
await sleep(200)
|
||||
await cdp.eval('window.__MO__.arm()')
|
||||
await sleep(tokens * intervalMs + 1500)
|
||||
}
|
||||
|
||||
const data = JSON.parse(await cdp.eval(COLLECT))
|
||||
|
||||
if (!real) {
|
||||
await cdp.eval('window.__PERF_DRIVE__.reset()')
|
||||
}
|
||||
|
||||
return analyze(data, real ? 0 : 500, { historyTurns })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
// Submit (Enter) latency + scroll stability. Subsumes measure-submit and
|
||||
// measure-jump. Backend tier: fires a REAL prompt, so run it on a throwaway
|
||||
// session with a live backend. Report-only (no committed baseline — real
|
||||
// round-trips are too environment-dependent to gate).
|
||||
|
||||
import { SELECTORS, sleep, typeIntoComposer } from '../lib/cdp.mjs'
|
||||
import { summarize } from '../lib/stats.mjs'
|
||||
|
||||
const MEASURE = `
|
||||
new Promise((resolve) => {
|
||||
const composer = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
||||
const thread = document.querySelector(${JSON.stringify(SELECTORS.threadContent)}) ||
|
||||
document.querySelector(${JSON.stringify(SELECTORS.threadViewport)})
|
||||
const viewport = document.querySelector(${JSON.stringify(SELECTORS.threadViewport)})
|
||||
const startCount = thread ? thread.querySelectorAll(${JSON.stringify(SELECTORS.turnPair)}).length : 0
|
||||
const startScroll = viewport ? viewport.scrollTop : 0
|
||||
const m = { start: performance.now(), maxJumpPx: 0 }
|
||||
let done = false
|
||||
|
||||
const finish = (reason) => {
|
||||
if (done) return
|
||||
done = true
|
||||
clearTimeout(timer); composerObs.disconnect(); threadObs && threadObs.disconnect()
|
||||
m.reason = reason
|
||||
resolve(m)
|
||||
}
|
||||
|
||||
const composerObs = new MutationObserver(() => {
|
||||
if (!m.composerClearedMs && composer && composer.innerText.length === 0) {
|
||||
m.composerClearedMs = performance.now() - m.start
|
||||
}
|
||||
})
|
||||
composer && composerObs.observe(composer, { childList: true, subtree: true, characterData: true })
|
||||
|
||||
let threadObs = null
|
||||
if (thread) {
|
||||
threadObs = new MutationObserver(() => {
|
||||
if (viewport) m.maxJumpPx = Math.max(m.maxJumpPx, Math.abs(viewport.scrollTop - startScroll))
|
||||
const c = thread.querySelectorAll(${JSON.stringify(SELECTORS.turnPair)}).length
|
||||
if (!m.userMsgRenderedMs && c > startCount) {
|
||||
m.userMsgRenderedMs = performance.now() - m.start
|
||||
requestAnimationFrame(() => { m.userMsgPaintMs = performance.now() - m.start; finish('paint') })
|
||||
}
|
||||
})
|
||||
threadObs.observe(thread, { childList: true, subtree: true })
|
||||
}
|
||||
|
||||
const timer = setTimeout(() => finish('timeout'), 5000)
|
||||
composer && composer.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', bubbles: true, cancelable: true }))
|
||||
})
|
||||
`
|
||||
|
||||
export default {
|
||||
name: 'submit',
|
||||
tier: 'backend',
|
||||
description: 'Enter → composer cleared → user message painted, plus scroll jump.',
|
||||
async run(cdp, opts = {}) {
|
||||
const rounds = Number(opts.rounds ?? 3)
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const clears = []
|
||||
const paints = []
|
||||
const jumps = []
|
||||
|
||||
for (let i = 0; i < rounds; i++) {
|
||||
await typeIntoComposer(cdp, `perf submit round ${i} ${'x'.repeat(30)}`, { cps: 60 })
|
||||
await sleep(250)
|
||||
const m = await cdp.eval(MEASURE)
|
||||
|
||||
if (typeof m.composerClearedMs === 'number') clears.push(m.composerClearedMs)
|
||||
if (typeof m.userMsgPaintMs === 'number') paints.push(m.userMsgPaintMs)
|
||||
jumps.push(m.maxJumpPx ?? 0)
|
||||
|
||||
// Let the turn finish before the next round so they don't pile up.
|
||||
await sleep(4000)
|
||||
}
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
submit_clear_p95_ms: summarize(clears).p95,
|
||||
submit_paint_p95_ms: summarize(paints).p95,
|
||||
submit_scroll_jump_max_px: Math.max(0, ...jumps)
|
||||
},
|
||||
detail: { rounds, clears: summarize(clears), paints: summarize(paints) }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// Large-transcript mount cost. New scenario (no prior script measured this):
|
||||
// loads N synthetic turns of mixed markdown into $messages and records the
|
||||
// mount→paint time plus any longtasks the mount blocks the main thread with.
|
||||
// This is the "open a long session" path — a first-impression latency.
|
||||
|
||||
import { sleep } from '../lib/cdp.mjs'
|
||||
|
||||
const OBSERVE = `
|
||||
(() => {
|
||||
window.__TM__ = { longtasks: [] }
|
||||
try {
|
||||
const po = new PerformanceObserver((l) => {
|
||||
for (const e of l.getEntries()) window.__TM__.longtasks.push(e.duration)
|
||||
})
|
||||
po.observe({ entryTypes: ['longtask'] })
|
||||
window.__TM__.po = po
|
||||
} catch {}
|
||||
return 'observing'
|
||||
})()
|
||||
`
|
||||
|
||||
export default {
|
||||
name: 'transcript',
|
||||
tier: 'ci',
|
||||
description: 'Mount + paint cost of loading a long transcript.',
|
||||
async run(cdp, opts = {}) {
|
||||
const turns = Number(opts.turns ?? 200)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
await cdp.eval(OBSERVE)
|
||||
|
||||
const mountMs = await cdp.eval(`window.__PERF_DRIVE__.loadTranscript(${turns})`)
|
||||
|
||||
// Let post-mount longtasks (content-visibility passes, virtualizer) settle.
|
||||
await sleep(1500)
|
||||
|
||||
const longtasks = await cdp.eval('window.__TM__.longtasks')
|
||||
await cdp.eval('try { window.__TM__.po && window.__TM__.po.disconnect() } catch {}')
|
||||
await cdp.eval('window.__PERF_DRIVE__.reset()')
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
transcript_mount_ms: Math.round(mountMs * 10) / 10,
|
||||
transcript_longtask_ms: Math.round(longtasks.reduce((a, b) => a + b, 0) * 10) / 10,
|
||||
transcript_longtask_max_ms: Math.round((longtasks.length ? Math.max(...longtasks) : 0) * 10) / 10
|
||||
},
|
||||
detail: { turns, messages: turns * 2, longtasks: longtasks.length }
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user