Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,134 @@
|
||||
{
|
||||
"summary": {
|
||||
"policy": "codex_real",
|
||||
"recall_pct": 26.7,
|
||||
"scores": [
|
||||
2,
|
||||
0,
|
||||
2,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
"read_seconds": 420,
|
||||
"compaction_events": 1,
|
||||
"peak_context_tokens": 482248,
|
||||
"rollout": "/home/teknium/.codex/sessions/2026/08/15/rollout-2026-08-15T18-20-54-01a00828-4e75-70f0-9e6d-31c7aac8d71c.jsonl"
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"q": "What is the reason given for never using 'git checkout pr-branch -- <file>' on stale branches?",
|
||||
"gold": "the stale file version silently deletes newer main code",
|
||||
"answer": "It can silently replace current files with stale branch versions, discarding newer fixes and resurrecting deleted code; salvage must use commits and resolve conflicts explicitly.",
|
||||
"score": 2,
|
||||
"why": "The answer accurately reflects the gold answer's point that stale branch versions will overwrite newer code, while adding specific detail about resurrecting deleted code."
|
||||
},
|
||||
{
|
||||
"q": "According to the transcript, how much RSS memory does the gateway balloon to every ~2h in the regression reported in issue #81625?",
|
||||
"gold": "~60GB",
|
||||
"answer": "UNKNOWN \u2014 best guess: approximately 4 GB RSS.",
|
||||
"score": 0,
|
||||
"why": "The answer provides a 'best guess' of 4 GB, which is factually incorrect and contradicts the gold answer of 60 GB."
|
||||
},
|
||||
{
|
||||
"q": "Which specific Electron setting is suspected of causing the Windows occlusion freeze in issue #83420?",
|
||||
"gold": "backgroundThrottling",
|
||||
"answer": "UNKNOWN \u2014 best guess: Electron\u2019s `backgroundThrottling: false`.",
|
||||
"score": 2,
|
||||
"why": "The answer correctly identifies the specific setting 'backgroundThrottling' as the suspected cause, despite the initial 'UNKNOWN' disclaimer."
|
||||
},
|
||||
{
|
||||
"q": "What exact error message is returned when 'gh pr merge --auto' is attempted on the NousResearch/hermes-agent repository?",
|
||||
"gold": "Auto merge is not allowed for this repository (enablePullRequestAutoMerge)",
|
||||
"answer": "`GraphQL: Pull request Auto merge is not allowed for this repository`",
|
||||
"score": 2,
|
||||
"why": "The answer matches the core message of the gold answer. The 'GraphQL:' prefix accurately reflects the technical origin of the error message when using the GitHub CLI."
|
||||
},
|
||||
{
|
||||
"q": "What is the specified 'Rule 0' that must be included in a subagent brief?",
|
||||
"gold": "load the skill first",
|
||||
"answer": "Rule 0: do not merge anything; only the primary agent/user is authorized to merge.",
|
||||
"score": 0,
|
||||
"why": "The answer identifies a different 'Rule 0' regarding merging permissions, which does not match the gold answer's requirement to 'load the skill first'."
|
||||
},
|
||||
{
|
||||
"q": "In the July 2026 title-cluster sweep, what was the title of the missed first submitter PR #35416?",
|
||||
"gold": "add config gate for title generation",
|
||||
"answer": "UNKNOWN \u2014 best guess: \u201cfix: prevent duplicate message submission.\u201d",
|
||||
"score": 0,
|
||||
"why": "The answer provides an incorrect guess ('fix: prevent duplicate message submission.') which does not match the gold answer ('add config gate for title generation')."
|
||||
},
|
||||
{
|
||||
"q": "Which file path is noted as containing the #34034/#28149 manifest guard 'test_bundled_plugin_manifests_ship_in_both_wheel_and_sdist'?",
|
||||
"gold": "tests/test_packaging_metadata.py",
|
||||
"answer": "UNKNOWN \u2014 best guess: `tests/test_plugin_packaging.py`.",
|
||||
"score": 0,
|
||||
"why": "The answer identifies the wrong file (test_plugin_packaging.py vs the correct test_packaging_metadata.py)."
|
||||
},
|
||||
{
|
||||
"q": "What was the result of the 'npm ci' command run in /home/teknium/salv-desktop according to the background process notification?",
|
||||
"gold": "completed normally (exit code 0)",
|
||||
"answer": "`npm ci` completed successfully with exit code 0.",
|
||||
"score": 2,
|
||||
"why": "The answer matches the gold answer perfectly, confirming the successful completion and the exit code of 0."
|
||||
},
|
||||
{
|
||||
"q": "What was the 'Root Cause A' identified for why 'uv sync --extra all --locked' failed daily in issue #79434?",
|
||||
"gold": "relative exclude-newer makes the committed lock stale every day",
|
||||
"answer": "Root Cause A: the `all` extra referenced platform-specific/nonexistent dependency groups that were absent or inconsistent in the locked resolution.",
|
||||
"score": 0,
|
||||
"why": "The answer identifies an issue with dependency groups, whereas the gold answer states the root cause was the use of a relative 'exclude-newer' value that caused the lockfile to become stale daily."
|
||||
},
|
||||
{
|
||||
"q": "How many tasks are reported as done in the 'fangliquanflq' desktop retry truncation PR #86605?",
|
||||
"gold": "13",
|
||||
"answer": "UNKNOWN \u2014 best guess: 7 tasks done.",
|
||||
"score": 0,
|
||||
"why": "The answer guesses 7 tasks, while the gold answer is 13. The answer is factually incorrect."
|
||||
},
|
||||
{
|
||||
"q": "In the 'salv-cron' worktree, what was the exit code when the agent tried to execute a 'BLOCKED (hardline)' command?",
|
||||
"gold": "-1",
|
||||
"answer": "Exit code 2.",
|
||||
"score": 0,
|
||||
"why": "The answer provides a specific exit code (2) that contradicts the gold answer (-1)."
|
||||
},
|
||||
{
|
||||
"q": "What is the full title block text for the technical schematic infographic generated for the Gateway Drain?",
|
||||
"gold": "GATEWAY DRAIN \u00d7 CRON \u2014 SHUTDOWN CONTRACT",
|
||||
"answer": "UNKNOWN \u2014 best guess: `GATEWAY DRAIN` / `INTERRUPT \u2192 DELIVER \u2192 DISCONNECT` / `GRACEFUL SHUTDOWN WITHOUT MESSAGE LOSS`.",
|
||||
"score": 0,
|
||||
"why": "The answer fails to provide the correct title block text specified in the gold answer. It incorrectly guesses metadata or slogans instead of the specific title 'GATEWAY DRAIN \u00d7 CRON \u2014 SHUTDOWN CONTRACT'."
|
||||
},
|
||||
{
|
||||
"q": "Which PR number's watcher reported '=== ALL GREEN (streak=1, checks=46) ===' at [03:56:19]?",
|
||||
"gold": "82980",
|
||||
"answer": "UNKNOWN \u2014 best guess: PR #86594.",
|
||||
"score": 0,
|
||||
"why": "The answer identifies the wrong PR number (86594 instead of 82980)."
|
||||
},
|
||||
{
|
||||
"q": "What is the specific Gist ID created for the PR infographic host in the cron cluster?",
|
||||
"gold": "ee33edd5804689243f974536ef7aecb9",
|
||||
"answer": "UNKNOWN \u2014 I do not retain the Gist ID.",
|
||||
"score": 0,
|
||||
"why": "The answer states it does not know the information, whereas the gold answer provides a specific Gist ID."
|
||||
},
|
||||
{
|
||||
"q": "What was the final merge SHA for Cluster D's Trigger-now PR #70638?",
|
||||
"gold": "f9d64b9a9d8b306f64851c1a13869d96ad5d7869",
|
||||
"answer": "UNKNOWN \u2014 I do not retain the final merge SHA for PR #70638.",
|
||||
"score": 0,
|
||||
"why": "The answer claims it does not know the information, while the gold answer provides the specific SHA requested."
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user