Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

2026-09-05 13:26:46 +03:00
commit 03634b1ca3
11340 changed files with 3442369 additions and 0 deletions
@@ -0,0 +1,84 @@
+// Time-to-first-token — Enter → first assistant token painted. The latency an
+// agent app is uniquely judged on, spanning the desktop submit path AND the
+// backend/agent-loop first-token time. Backend tier: fires a REAL prompt, needs
+// a live backend (and credits). Report-only.
+//
+//   node scripts/perf/run.mjs first-token --spawn --prompt "hi"
+
+import { SELECTORS, sleep, typeIntoComposer } from '../lib/cdp.mjs'
+import { summarize } from '../lib/stats.mjs'
+
+export default {
+  name: 'first-token',
+  tier: 'backend',
+  description: 'Enter → first assistant token painted (real backend).',
+  async run(cdp, opts = {}) {
+    const rounds = Number(opts.rounds ?? 3)
+    const prompt = opts.prompt ?? 'reply with a single short sentence'
+    const timeoutMs = Number(opts.timeoutMs ?? 60000)
+
+    await cdp.send('Runtime.enable')
+
+    const firstTokens = []
+
+    for (let i = 0; i < rounds; i++) {
+      const baseText = await cdp.eval(`(() => {
+        const a = document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)})
+        return a.length ? a[a.length - 1].textContent.length : 0
+      })()`)
+      const baseCount = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`)
+
+      await typeIntoComposer(cdp, `${prompt} (${i})`, { cps: 60 })
+      const submitAt = Date.now()
+      await cdp.eval(`(() => {
+        const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
+        el && el.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', bubbles: true, cancelable: true }))
+      })()`)
+
+      const deadline = Date.now() + timeoutMs
+      let firstTokenMs = null
+
+      while (Date.now() < deadline) {
+        await sleep(25)
+        const grown = await cdp.eval(`(() => {
+          const a = document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)})
+          if (a.length > ${baseCount}) return true
+          return a.length ? a[a.length - 1].textContent.length > ${baseText} : false
+        })()`)
+
+        if (grown) {
+          firstTokenMs = Date.now() - submitAt
+          break
+        }
+      }
+
+      if (firstTokenMs !== null) {
+        firstTokens.push(firstTokenMs)
+      }
+
+      // Let the turn finish before the next round.
+      const turnDeadline = Date.now() + timeoutMs
+      while (Date.now() < turnDeadline) {
+        await sleep(250)
+        const busy = await cdp.eval(`!!document.querySelector('[data-status="running"], [data-busy="true"]')`)
+
+        if (!busy) {
+          break
+        }
+      }
+
+      await sleep(500)
+    }
+
+    if (!firstTokens.length) {
+      throw new Error('no first token observed — is a backend with credits connected?')
+    }
+
+    const s = summarize(firstTokens)
+
+    return {
+      metrics: { first_token_p50_ms: s.p50, first_token_p95_ms: s.p95 },
+      detail: { rounds, samples: firstTokens, summary: s }
+    }
+  }
+}