[
  {
    "id": "databricks",
    "name": "Databricks coding-agent benchmark",
    "publisher": "Databricks",
    "published": "2026-07-08",
    "date_basis": "Article publication date",
    "task_surface": "Recent human-written pull requests from a multi-million-line internal codebase",
    "task_count": null,
    "matched_model_comparison": true,
    "includes_pi": true,
    "reports_cost": true,
    "repeated_trials": null,
    "grader": "Repository tests reviewed by engineers",
    "data_access": "Methods and aggregate claims only",
    "conflict": "none",
    "conflict_note": "Databricks evaluated third-party harnesses for its own engineering use and ships none of the compared systems.",
    "weight_tier": "anchor",
    "weight_rationale": "A buyer study on real pull requests from a multi-million-line production codebase, with matched models and broad scope.",
    "citation_note": "Directional anchor. The source publishes aggregate claims only, so cite direction and rough magnitude rather than exact points.",
    "dispersion_pp": 3.5,
    "dispersion_note": "No CI or task count is published, so the winning interval uses the study-internal noise scale: the publisher treats results a couple of points apart as equivalent and a 6-point gap (Sonnet 5 vs Opus 4.8) as material, and quality wiggle between adjacent effort settings of the same model-harness pair runs 1 to 8 points (RMS about 4, an upper bound since effort has a real effect). ±3.5 pp per observation, about ±5 on a pair, encodes that reading: a 6-point gap clears it, a 2-3 point gap does not.",
    "source_url": "https://www.databricks.com/blog/benchmarking-coding-agents-databricks-multi-million-line-codebase",
    "section": "Operational and broad evidence",
    "section_order": 1,
    "overview_summary": "Across 6 matched comparisons, Claude Code or Codex cost 1.20 to 2.08 times as much as Pi. Quality stayed within 7 points.",
    "slide_lead": "Databricks ran 6 matched harness comparisons: Claude Code or Codex cost 1.20 to 2.08 times as much as Pi, with quality within 7 points.",
    "method_summary": "Databricks replayed recent human-written pull requests from its multi-million-line internal codebase. It compared Claude Code with Pi on Opus 4.8 and Codex with Pi on GPT-5.5 at matched effort settings.",
    "conclusion": "Pi was cheaper in all 6 published pairs. It also re-fed less context: 236,000 versus 742,000 tokens on Opus 4.8 and 665,000 versus 1.235 million on GPT-5.5.",
    "limitation": "The source does not publish task count, run-level data or uncertainty. Per-task dollar costs: three combinations are stated exactly in the text; the rest are derived from the published cost ratios or traced from Figure 1."
  },
  {
    "id": "wandr",
    "name": "WANDR system comparison and delivery ablation",
    "publisher": "Perplexity",
    "published": "2026-07-14",
    "date_basis": "Article and technical report publication date",
    "task_surface": "Wide-and-deep web research requiring large, evidence-backed collections",
    "task_count": 500,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": false,
    "grader": "WANDR record-level verifier with pinned GPT-5.4 judges",
    "data_access": "Public paper, 500 task packages, evaluation code and aggregate tables; no evaluated run artefacts",
    "conflict": "compared vendor",
    "conflict_note": "Perplexity publishes Search as Code, one of the compared systems.",
    "weight_tier": "supporting",
    "weight_rationale": "The within-system delivery ablations isolate a named runtime component across 45 tasks; the 500-task production-system comparison is contextual because provider routes, tools and search infrastructure differ.",
    "source_url": "https://research.perplexity.ai/articles/wandr-benchmark-evaluating-research-agents-that-must-search-wide-and-deep",
    "section": "Operational and broad evidence",
    "section_order": 9,
    "overview_summary": "A contextual production-system comparison showed a large quality-cost trade-off; within-system ablations favoured file or share delivery over output tokens.",
    "slide_lead": "WANDR compared 500 research runs across 2 production systems, then isolated output delivery on 45 tasks: file or share delivery scored higher in all 3 systems.",
    "method_summary": "Perplexity compared Search as Code and OpenAI Responses, both labelled GPT-5.5, across 500 tasks and a 45-task effort sweep. Separately, it held each provider route and effort fixed while changing only whether results were delivered through files or output tokens.",
    "conclusion": "The production systems showed a large quality-cost trade-off but do not isolate a harness effect. Within each of Perplexity, Anthropic and OpenAI, file or share delivery scored above output-token delivery.",
    "limitation": "The production systems may use different model snapshots, search infrastructure and tools. The paper reports one rollout per task, no confidence intervals and no evaluated run artefacts; delivery-ablation arms are independent rather than paired rollouts.",
    "citation_note": "Treat Search as Code versus OpenAI Responses as a same-label production-system comparison, not an isolated harness effect. The within-system output-delivery ablations are supporting component evidence, but use independent rollouts."
  },
  {
    "id": "warden",
    "name": "Warden benchmark",
    "publisher": "Sentry",
    "published": "2026-06-05",
    "date_basis": "Publication date of the matched Pi and Claude SDK comparisons used here",
    "task_surface": "Known-vulnerability recognition in files tied to historical Sentry vulnerabilities",
    "task_count": 86,
    "matched_model_comparison": true,
    "includes_pi": true,
    "reports_cost": true,
    "repeated_trials": false,
    "grader": "Known-corpus recall",
    "data_access": "Public result table",
    "conflict": "none",
    "weight_tier": "supporting",
    "weight_rationale": "Real historical vulnerabilities in Sentry code, but the grader measures recognition of known vulnerabilities in one domain, without repeated trials.",
    "citation_note": "Cite the comparator as Warden's Claude Agent SDK configuration, not as Claude Code. The Agent SDK shares Claude Code's runtime and tools, but Warden replaces the native system prompt with its own, restricts the agent to read-only tools and disables subagents, shell and edits (checked 18 July 2026).",
    "source_url": "https://warden.sentry.dev/benchmarking/",
    "section": "Operational and broad evidence",
    "section_order": 10,
    "overview_summary": "Pi found a similar or larger share of known vulnerabilities at about one-quarter to one-fifth of Claude SDK's cost.",
    "slide_lead": "Sentry tested 86 known vulnerabilities: Pi found a similar or larger share while costing about one-quarter to one-fifth as much as Claude SDK.",
    "method_summary": "Sentry gave agents files connected to 86 historical vulnerabilities and measured how many known findings they recovered. The published table compares matched Claude models running in Pi and Claude SDK.",
    "conclusion": "Pi cost less and found a similar or larger share of the known corpus for the matched Claude models.",
    "limitation": "The comparator is Claude SDK, not the Claude Code command-line product. The benchmark does not measure whole-repository discovery."
  },
  {
    "id": "harness-optimization-lab",
    "name": "Harness optimization on LAB",
    "publisher": "Joel Niklaus, Hugging Face",
    "published": "2026-07-01",
    "date_basis": "Research article publication date",
    "task_surface": "Realistic legal matters requiring evidence-backed deliverables",
    "task_count": 100,
    "matched_model_comparison": true,
    "includes_pi": true,
    "reports_cost": true,
    "repeated_trials": false,
    "grader": "LAB criterion rubrics scored by Claude Sonnet 4.6",
    "data_access": "Public code, frozen split, run results and proposer traces",
    "conflict": "compared vendor",
    "conflict_note": "The study author built and released the optimized LAB harness, the strongest compared system.",
    "weight_tier": "supporting",
    "weight_rationale": "A 100-task held-out matched-model comparison with public artefacts, but one test rollout per configuration, an LLM judge and a study author who built the leading harness.",
    "source_url": "https://huggingface.co/spaces/joelniklaus/harness-optimization",
    "section": "Focused domain evidence",
    "section_order": 15,
    "overview_summary": "With DeepSeek V4 Pro fixed, pooled criterion pass rate ranged from 3.5% to 80.1% across 5 harnesses.",
    "slide_lead": "LAB held DeepSeek V4 Pro fixed on 100 legal tasks: changing the harness moved pooled criterion pass rate from 3.5% to 80.1%.",
    "method_summary": "The study froze DeepSeek V4 Pro, a 100-task held-out LAB split and the Sonnet 4.6 judge. It compared an automatically optimized LAB harness, vanilla LAB, Pi, Goose and mini-swe-agent.",
    "conclusion": "Harness choice dominated the measured result. Deterministic file-delivery and runtime fixes produced the largest gains, while general coding harnesses struggled with LAB's deliverable contract.",
    "limitation": "The judge is model-based, test configurations were run once, and the optimized harness was selected after repeated development on a separate 24-task split. Timeouts and output-delivery failures remained scored as harness failures."
  },
  {
    "id": "latchbio-scbench-long",
    "name": "LatchBio behavioural-prior study",
    "publisher": "LatchBio",
    "published": "2026-07-10",
    "date_basis": "Harness-analysis article publication date",
    "task_surface": "Short-horizon preclinical pharmacology and long biological analysis workflows",
    "task_count": 121,
    "matched_model_comparison": true,
    "includes_pi": true,
    "reports_cost": true,
    "repeated_trials": true,
    "grader": "Deterministic biological graders",
    "data_access": "Public benchmark JSON and trajectories; aggregate prompt-ablation table",
    "conflict": "adjacent",
    "conflict_note": "LatchBio sells a bioinformatics platform in the benchmark's domain but ships none of the compared harnesses.",
    "weight_tier": "anchor",
    "weight_rationale": "Two fixed-model suites cover 121 biological tasks with 3 trials, deterministic graders and public standard-harness results; the prompt ablations are aggregate-only.",
    "citation_note": "The prompt-ablation arms publish exact aggregate deltas but no linked task-level results. Treat scBench-Long's 21-task intervals as uncertain even though the combined study is an anchor.",
    "source_url": "https://blog.latch.bio/p/why-an-open-source-harness-outperforms",
    "section": "Focused domain evidence",
    "section_order": 16,
    "overview_summary": "Point-estimate ordering changed by task horizon, although the harness intervals overlapped; a prompt-only ablation moved in opposite directions.",
    "slide_lead": "LatchBio repeated 121 biology tasks: Pi's point estimate was 4.6 points higher on shorter work, while Claude Code's was 4.8 points higher on long workflows; intervals overlapped.",
    "method_summary": "LatchBio ran 100 TxBench-PP tasks and 21 scBench-Long workflows 3 times per configuration. It compared Pi with native harnesses, then held Opus 4.8, Pi's tools, tasks and graders fixed while adding calibration and orchestration instructions to Pi's system prompt.",
    "conclusion": "The point-estimate ordering changed with task horizon, but published harness intervals overlapped on both suites. In the prompt-only ablation, orchestration guidance reduced TxBench-PP pass rate by 4.7 points and increased scBench-Long pass rate by 3.2 points.",
    "limitation": "The harness comparisons do not establish a winner because their intervals overlap, and the 21-task long-horizon intervals are especially wide. Prompt-ablation results are aggregate-only, and the behavioural clauses approximate rather than reproduce Claude Code's full prompt."
  },
  {
    "id": "openbench",
    "name": "OpenBench M4.5",
    "publisher": "OpenBench",
    "published": "2026-07-03",
    "date_basis": "Public release date of the M4.5 results used here",
    "task_surface": "Three repository coding tasks with executable checkers",
    "task_count": 3,
    "matched_model_comparison": true,
    "includes_pi": true,
    "reports_cost": false,
    "repeated_trials": true,
    "grader": "Executable checkers with partial-credit support",
    "data_access": "Code and committed result data",
    "conflict": "none",
    "weight_tier": "contextual",
    "weight_rationale": "Three tasks that every harness passed cannot separate frontier harnesses on correctness.",
    "source_url": "https://github.com/minghinmatthewlam/openbench",
    "section": "Small and diagnostic evidence",
    "section_order": 26,
    "overview_summary": "All 4 harnesses passed every task. Pi and Cursor used less time and fewer tokens.",
    "slide_lead": "OpenBench repeated 3 coding tasks: every harness passed, so the useful difference was efficiency rather than correctness.",
    "method_summary": "OpenBench ran 3 repository coding tasks 3 times with GPT-5.5 at medium effort. Executable checkers awarded partial credit, while the study also recorded wall time and token use.",
    "conclusion": "All clean harnesses passed every task. Pi and Cursor used less time and fewer tokens.",
    "limitation": "The tasks were too easy to separate frontier harnesses on correctness."
  },
  {
    "id": "drugdiscoverybench",
    "name": "DrugDiscoveryBench",
    "publisher": "Scale AI",
    "published": "2026-06-29",
    "date_basis": "Initial public repository release",
    "task_surface": "Expert-curated drug-discovery and life-sciences workflows",
    "task_count": 82,
    "matched_model_comparison": true,
    "includes_pi": true,
    "reports_cost": true,
    "repeated_trials": true,
    "grader": "Expert-authored rubrics scored by an LLM judge",
    "data_access": "Tasks are public; rubrics and reference answers require access approval",
    "conflict": "adjacent",
    "conflict_note": "Scale AI sells evaluation and data services but ships none of the compared harnesses.",
    "weight_tier": "supporting",
    "weight_rationale": "Expert-curated tasks with repeated trials, but gated rubrics and an outcome that depends on an LLM judge.",
    "source_url": "https://github.com/scaleapi/DrugDiscoveryBench",
    "section": "Operational and broad evidence",
    "section_order": 6,
    "overview_summary": "mini-SWE-agent scored highest with GPT-5.5. Codex placed in the middle.",
    "slide_lead": "DrugDiscoveryBench repeated 82 expert workflows: mini-SWE-agent led the matched GPT-5.5 comparison, while Codex placed in the middle.",
    "method_summary": "Scale AI ran 82 expert-curated drug-discovery and life-sciences workflows 3 times. An LLM judge scored each result against expert-authored rubrics and reference answers.",
    "conclusion": "GPT-5.5 performed best in mini-SWE-agent. Codex was neither the best nor worst harness for that model.",
    "limitation": "The paper plots cost but does not publish a table of exact cost points. The outcome depends on an LLM judge."
  },
  {
    "id": "artificial-analysis",
    "name": "Coding Agent Index",
    "publisher": "Artificial Analysis",
    "published": "2026-05-11",
    "date_basis": "Coding Agent Index launch date",
    "task_surface": "321 tasks across DeepSWE, Terminal-Bench v2 and SWE-Atlas-QnA",
    "task_count": 321,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": true,
    "grader": "Program checks, test suites and rubric scoring",
    "data_access": "Public methods and live results",
    "conflict": "none",
    "conflict_note": "All 3 component suites are externally produced and Artificial Analysis ships none of the compared harnesses (checked 18 July 2026).",
    "weight_tier": "anchor",
    "weight_rationale": "An independent evaluator running 321 tasks from 3 externally produced suites with matched models and repeated runs.",
    "citation_note": "Directional anchor. No uncertainty or task-level results are published and the live suite changes, so cite direction and rough magnitude rather than exact points. Terminal-Bench v2 results overlap the Terminal-Bench study.",
    "component_suites": [
      {
        "name": "DeepSWE",
        "producer": "Datacurve",
        "task_count": 113,
        "provenance": "Human-written tasks authored from scratch on real open-source repositories",
        "overlaps_study": null
      },
      {
        "name": "Terminal-Bench v2",
        "producer": "Laude Institute",
        "task_count": 84,
        "provenance": "Crowdsourced human-written command-line tasks; 84 of the original 89 tasks are used",
        "overlaps_study": "terminal-bench"
      },
      {
        "name": "SWE-Atlas-QnA",
        "producer": "Scale AI",
        "task_count": 124,
        "provenance": "Expert-written question-answering tasks over real production repositories",
        "overlaps_study": null
      }
    ],
    "source_url": "https://artificialanalysis.ai/agents/coding-agents",
    "section": "Operational and broad evidence",
    "section_order": 4,
    "overview_summary": "OpenCode led with Opus 4.7; Codex led Cursor with GPT-5.5. In both comparisons, the higher-scoring harness cost more.",
    "slide_lead": "Artificial Analysis ran 321 tasks 3 times: the higher-scoring harness depended on the model and cost more in both full-suite comparisons.",
    "method_summary": "Artificial Analysis combines 321 tasks from DeepSWE, Terminal-Bench v2 and SWE-Atlas-QnA. It repeats each configuration 3 times and reports an equally weighted index, API cost and active wall time.",
    "conclusion": "OpenCode led the Opus 4.7 group by 7.7 points; Codex led Cursor with GPT-5.5 by 8.7 points. Most of both gaps came from DeepSWE.",
    "limitation": "No uncertainty or task-level results are published. The live suite, pricing and harness versions change. The GPT-5.5 match relies on the published model and effort labels."
  },
  {
    "id": "endor",
    "name": "Agent Security League",
    "publisher": "Endor Labs",
    "published": "2026-07-09",
    "date_basis": "Publication date of the latest matched-harness comparison used here",
    "task_surface": "Real vulnerability-fixing tasks with hidden functional and security tests",
    "task_count": 200,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": false,
    "repeated_trials": false,
    "grader": "Hidden functional and security tests",
    "data_access": "Public articles and leaderboard",
    "conflict": "adjacent",
    "conflict_note": "Endor Labs ships AURI, a security layer for coding agents, not a compared harness. The benchmark page markets AURI and Endor Labs has a disclosed partnership with Cursor, a compared vendor (checked 18 July 2026).",
    "weight_tier": "supporting",
    "weight_rationale": "200 real vulnerability-fixing tasks with hidden functional and security tests, but one run per configuration and a publisher with a commercial stake in the findings.",
    "source_url": "https://www.endorlabs.com/learn/claude-sonnet-5-with-cursor-strong-reasoning-throttled-by-the-harness",
    "section": "Operational and broad evidence",
    "section_order": 11,
    "overview_summary": "Cursor led on Opus 4.7 and 4.8. Claude Code led on Sonnet 5.",
    "slide_lead": "Endor Labs tested 200 vulnerability repairs: Cursor led for Opus, but Claude Code led for Sonnet 5.",
    "method_summary": "Endor Labs asked each configuration to repair 200 real vulnerabilities. Hidden tests separately measured whether the software still worked and whether the vulnerability was fixed.",
    "conclusion": "Cursor led Claude Code for Opus 4.7 and 4.8. Claude Code led Cursor for Sonnet 5.",
    "limitation": "There are no repeated trials or task-level costs. Timeouts explain part of the reversal."
  },
  {
    "id": "xor",
    "name": "Vulnerability-Agent-Bench",
    "publisher": "XOR",
    "published": "2026-02-01",
    "date_basis": "Benchmark page publication date in the publisher sitemap",
    "task_surface": "Real vulnerability repairs in isolated containers",
    "task_count": 136,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": false,
    "grader": "Vulnerability repair checks",
    "data_access": "Public explorer and linked traces",
    "conflict": "adjacent",
    "conflict_note": "XOR is a security company publishing in its own market but ships none of the compared harnesses.",
    "weight_tier": "supporting",
    "weight_rationale": "Real vulnerability repairs with public traces, but single runs and an inconsistent published sample count.",
    "source_url": "https://www.xor.tech/resources/benchmarks/agents",
    "section": "Focused domain evidence",
    "section_order": 18,
    "overview_summary": "Claude Code and Codex beat OpenCode on all 3 published matched-model pairs.",
    "slide_lead": "XOR tested 136 vulnerability repairs: Claude Code and Codex beat OpenCode on all 3 published matched-model pairs.",
    "method_summary": "XOR ran agents in isolated containers on real vulnerability-repair tasks. Its public explorer reports pass rate and cost for matched model and harness configurations.",
    "conclusion": "Claude Code and Codex led the corresponding OpenCode configurations on the published matched pairs.",
    "limitation": "The site alternates between 128 and 136 samples. Some reported cost differences need more explanation."
  },
  {
    "id": "wildclawbench",
    "name": "WildClawBench",
    "publisher": "InternLM",
    "published": "2026-05-11",
    "date_basis": "arXiv v1 submission date",
    "task_surface": "Bilingual, multimodal and long-horizon agent tasks",
    "task_count": 60,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": null,
    "grader": "Deterministic checks, environment checks and model judges",
    "data_access": "Code, tasks, images and evaluation details",
    "conflict": "none",
    "conflict_note": "Hermes Agent is a Nous Research project and OpenClaw is a community project. The matched-model rows use GPT-5.4, GLM 5, MiMo V2 Pro and MiniMax M2.7, none of which is an InternLM model (checked 18 July 2026).",
    "weight_tier": "supporting",
    "weight_rationale": "Matched models across 4 harnesses on a broad task mix, but each harness image has different tools and prompt conventions and repeat counts are unclear.",
    "source_url": "https://github.com/InternLM/WildClawBench",
    "section": "Focused domain evidence",
    "section_order": 19,
    "overview_summary": "The best harness changed by model. Switching harness moved scores by double digits.",
    "slide_lead": "WildClawBench tested 60 long-horizon tasks: changing the harness produced double-digit score shifts and changed the winner by model.",
    "method_summary": "InternLM evaluated 60 bilingual, multimodal and long-horizon tasks in harness-specific container images. Deterministic checks, environment checks and model judges contributed to the overall score.",
    "conclusion": "The best harness changed with the model. The source reports double-digit score changes from switching harness.",
    "limitation": "Each harness image has different tools and prompt conventions. Repeat counts are not clear in the headline table."
  },
  {
    "id": "harnessbench",
    "name": "HarnessBench",
    "publisher": "Nyosegawa",
    "published": "2026-05-08",
    "date_basis": "Article publication date",
    "task_surface": "Recent bug-fix issues from 9 open-source repositories",
    "task_count": 27,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": false,
    "grader": "Hidden core and regression tests",
    "data_access": "Code and public results",
    "conflict": "none",
    "weight_tier": "supporting",
    "weight_rationale": "Recent real bug-fix issues with public code and results. The author reports the 27-task pass-rate differences as not statistically significant, so the study supports uncertainty claims rather than rankings.",
    "source_url": "https://github.com/nyosegawa/harness-bench",
    "section": "Small and diagnostic evidence",
    "section_order": 24,
    "overview_summary": "Several configurations finished within one task of the leader. Runtime differed more clearly.",
    "slide_lead": "HarnessBench tested 27 recent bug fixes: several configurations finished within one task of the leader, but runtime differed more clearly.",
    "method_summary": "HarnessBench selected 27 recent bug-fix issues from 9 open-source repositories. Hidden tests checked the core fix and regressions, with one run for each configuration.",
    "conclusion": "Several conditions finished within one task of the observed leader. Runtime differences were clearer than pass-rate differences.",
    "limitation": "The author states that the 27-task pass-rate differences are not statistically significant."
  },
  {
    "id": "corebench-hard",
    "name": "CORE-Bench Hard",
    "publisher": "Princeton HAL",
    "published": "2026-06-23",
    "date_basis": "Publication date of the study covering the updated scaffold comparison",
    "task_surface": "Scientific reproducibility tasks",
    "task_count": 45,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": false,
    "grader": "Research artifact reproduction",
    "data_access": "Leaderboard and encrypted traces",
    "conflict": "none",
    "weight_tier": "supporting",
    "weight_rationale": "An independent leaderboard on 45 reproducibility tasks, but one run per configuration and compared systems that differ beyond the harness.",
    "source_url": "https://hal.cs.princeton.edu/corebench_hard",
    "section": "Focused domain evidence",
    "section_order": 20,
    "overview_summary": "Claude Code more than doubled CORE-Agent's verified success with Opus 4.5 while costing less.",
    "slide_lead": "CORE-Bench Hard tested 45 scientific reproductions: Claude Code more than doubled verified success while costing less than CORE-Agent.",
    "method_summary": "Princeton HAL asked agents to reproduce 45 research artefacts. The leaderboard verifies whether each system recreated the required scientific result and reports total cost.",
    "conclusion": "Claude Code more than doubled the verified success of CORE-Agent for Claude Opus 4.5 while costing less.",
    "limitation": "There is one run per configuration. The systems differ across retrieval, tools and context policy."
  },
  {
    "id": "acode-cli",
    "name": "A-CODE-CLI",
    "publisher": "AIMultiple",
    "published": "2026-06-29",
    "date_basis": "Benchmark article date",
    "task_surface": "Ten full-stack web development scenarios",
    "task_count": 10,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": true,
    "grader": "Backend checks and Playwright user-interface checks",
    "data_access": "Public report and one example task",
    "conflict": "none",
    "weight_tier": "contextual",
    "weight_rationale": "Ten scenarios with only one public example task; too small to weight beyond context.",
    "source_url": "https://aimultiple.com/agentic-cli",
    "section": "Small and diagnostic evidence",
    "section_order": 25,
    "overview_summary": "OpenCode had a small backend lead over Claude Code and cost 44% less with Sonnet 4.6.",
    "slide_lead": "A-CODE-CLI repeated 10 web scenarios: OpenCode had a 2.4-point backend lead and cost 44% less than Claude Code.",
    "method_summary": "AIMultiple ran 10 full-stack web development scenarios 3 times with Claude Sonnet 4.6. Backend checks and Playwright tests measured correctness and user-interface behaviour.",
    "conclusion": "OpenCode had a small backend lead over Claude Code and cost less in the matched Sonnet 4.6 comparison.",
    "limitation": "The benchmark has only 10 scenarios. Only one example task is public."
  },
  {
    "id": "portkey-harness-tax",
    "name": "Harness Tax",
    "publisher": "Portkey",
    "published": "2026-04-13",
    "date_basis": "Article publication date",
    "task_surface": "Two trivial prompts routed through a gateway",
    "task_count": 2,
    "matched_model_comparison": false,
    "includes_pi": true,
    "reports_cost": false,
    "repeated_trials": false,
    "grader": "Input-token capture",
    "data_access": "Method and aggregate values",
    "conflict": "adjacent",
    "conflict_note": "Portkey sells the gateway used to capture the measurements but ships none of the compared harnesses.",
    "weight_tier": "contextual",
    "weight_rationale": "A 2-prompt overhead diagnostic that measures fixed input tokens, not coding quality.",
    "source_url": "https://portkey.ai/blog/the-harness-tax/",
    "section": "Small and diagnostic evidence",
    "section_order": 27,
    "overview_summary": "Fixed input overhead ranged from about 2,600 tokens for Pi to 27,000 for Claude Code.",
    "slide_lead": "Portkey measured 2 trivial requests: fixed input overhead ranged from about 2,600 tokens for Pi to 27,000 for Claude Code.",
    "method_summary": "Portkey routed 2 trivial prompts through its gateway and recorded the input payload sent by each harness. The exercise isolates fixed prompt overhead rather than coding performance.",
    "conclusion": "The fixed input payload ranged from about 2,600 tokens for Pi to 27,000 for Claude Code.",
    "limitation": "This is an overhead diagnostic. It does not measure coding quality or full-session economics."
  },
  {
    "id": "github-copilot",
    "name": "GitHub Copilot harness evaluation",
    "publisher": "GitHub",
    "published": "2026-06-25",
    "date_basis": "Article publication date",
    "task_surface": "Five coding and terminal benchmark suites, including SWE-bench and Terminal-Bench 2",
    "task_count": null,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": true,
    "grader": "Public benchmark graders and internal executable checks",
    "data_access": "Methods and exact relative-result charts; no raw rows",
    "conflict": "compared vendor",
    "conflict_note": "GitHub ships Copilot CLI, one of the compared harnesses.",
    "weight_tier": "supporting",
    "weight_rationale": "A broad matched study with repeated runs, but published by a vendor of a compared harness and without raw result rows.",
    "citation_note": "Cite only suites with at least 100 tasks. For smaller suites the source reports the best of 5 runs while labelling the metric pass@1, so treat those figures as contextual.",
    "source_url": "https://github.blog/ai-and-ml/github-copilot/evaluating-performance-and-efficiency-of-the-github-copilot-agentic-harness-across-models-and-tasks/",
    "section": "Operational and broad evidence",
    "section_order": 2,
    "overview_summary": "Across 20 matched pairs, Copilot CLI moved task resolution by -8.0 to +7.1 points and used fewer median tokens in 18.",
    "slide_lead": "GitHub compared Copilot CLI with Claude Code or Codex across 5 suites: quality was mixed, while Copilot used fewer median tokens in 18 of 20 pairs.",
    "method_summary": "GitHub fixed the model, task, context limit, reasoning effort and environment across Copilot CLI and each model provider's native harness. It tested 4 models on 5 suites and ran each configuration 5 times.",
    "conclusion": "Task-resolution differences ranged from 8.0 points worse to 7.1 points better. Copilot used fewer median tokens in 18 of 20 model and benchmark pairs.",
    "limitation": "This is a vendor study with no raw result rows. For suites under 100 tasks, GitHub reports the best of 5 runs while labelling the metric pass@1."
  },
  {
    "id": "terminal-bench",
    "name": "Terminal-Bench 2.1",
    "publisher": "Laude Institute",
    "published": "2026-05-06",
    "date_basis": "Terminal-Bench 2.1 release date",
    "task_surface": "Containerized command-line tasks with deterministic final-state checks",
    "task_count": 89,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": true,
    "grader": "Deterministic final-state tests",
    "data_access": "Public leaderboard, paper, experiments and submissions",
    "conflict": "none",
    "weight_tier": "anchor",
    "weight_rationale": "An independent benchmark with a public paper, leaderboard, experiments and submissions, deterministic grading and repeated trials.",
    "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1",
    "section": "Operational and broad evidence",
    "section_order": 3,
    "overview_summary": "The native CLI led 4 of 6 current matched pairs; 1 was effectively tied and Terminus 2 led Gemini 3 Pro.",
    "slide_lead": "Terminal-Bench repeated 89 command-line tasks: the native CLI led 4 of 6 current matched pairs, but not for every model.",
    "method_summary": "Terminal-Bench runs 89 containerized terminal tasks with deterministic final-state tests. The current 2.1 leaderboard contains 6 clean same-model pairs between a provider CLI and Terminus 2, with at least 5 runs per configuration.",
    "conclusion": "Claude Code or Codex led 4 pairs. Gemini 3.1 Pro was effectively tied, while Terminus 2 led Gemini 3 Pro by 8.1 points.",
    "limitation": "The leaderboard changes over time. A token appendix in the 2.0 paper says it covers 74 tasks, not the full 89, so this review does not derive per-task token values from it."
  },
  {
    "id": "agents-last-exam",
    "name": "Agents' Last Exam",
    "publisher": "RDI Berkeley",
    "published": "2026-06-03",
    "date_basis": "arXiv v1 submission date",
    "task_surface": "Long-horizon professional workflows across CLI, GUI, files, applications and web research",
    "task_count": 152,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": null,
    "grader": "Task-specific executable and rubric-based checks",
    "data_access": "Public paper, tables, code and data",
    "conflict": "none",
    "weight_tier": "anchor",
    "weight_rationale": "An academic study of 152 long-horizon tasks across several domains, with matched models and public paper, code and data.",
    "source_url": "https://arxiv.org/abs/2606.05405",
    "section": "Operational and broad evidence",
    "section_order": 5,
    "overview_summary": "GPT-5.5 full-pass rates spanned 19.1% to 24.0%, while published total cost spanned $180 to $570.",
    "slide_lead": "Agents' Last Exam tested long professional workflows: GPT-5.5 harnesses stayed within 4.9 points, but total cost differed by more than 3 times.",
    "method_summary": "Agents' Last Exam evaluates long professional workflows with runs capped at 5 hours. It reports fixed-model harness sweeps for GPT-5.5 and Opus 4.7, plus a 105-task ALE-CLI sweep for Sonnet 4.6.",
    "conclusion": "The harness changed both full-pass rate and resources, but the model effect could be larger: GPT-5.5 harnesses were within 4.9 points while the fixed-OpenClaw model spread was 16.8 points.",
    "limitation": "Only some configurations have 3 independent runs. The published tier counts do not add cleanly to the stated 152-task public set."
  },
  {
    "id": "claw-swe-bench",
    "name": "Claw-SWE-Bench",
    "publisher": "OpenSquilla",
    "published": "2026-06-10",
    "date_basis": "arXiv v1 submission date",
    "task_surface": "Multilingual GitHub issue-resolution tasks",
    "task_count": 350,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": false,
    "grader": "Executable issue-resolution tests",
    "data_access": "Public paper, adapters, dataset and result artefacts",
    "conflict": "none",
    "weight_tier": "supporting",
    "weight_rationale": "350 real multilingual issue-resolution tasks with public artefacts, but one attempt per task and no confidence intervals.",
    "citation_note": "Treat token and cost values as contextual: the source's token accounting contains large harness-specific anomalies.",
    "source_url": "https://arxiv.org/abs/2606.12344",
    "section": "Operational and broad evidence",
    "section_order": 7,
    "overview_summary": "The harness spread was 12.5 points with GLM-5.1 and 27.4 points with Qwen3.6-Flash.",
    "slide_lead": "Claw-SWE-Bench crossed 5 harnesses with 2 models on 350 issues: the harness spread grew from 12.5 to 27.4 points depending on the model.",
    "method_summary": "Claw-SWE-Bench fully crossed 5 harnesses and 2 models on the same 350 multilingual GitHub issues, with a fixed prompt, workspace contract, runtime budget, patch extraction and evaluator.",
    "conclusion": "OpenClaw led both model groups. The gap between best and worst harness was 12.5 points for GLM-5.1 and 27.4 points for Qwen3.6-Flash.",
    "limitation": "There is one attempt per task and configuration, with no confidence intervals. Token accounting contains large harness-specific anomalies."
  },
  {
    "id": "harness-bench-paper",
    "name": "Harness-Bench paper",
    "publisher": "Qihoo 360",
    "published": "2026-05-27",
    "date_basis": "arXiv v1 submission date",
    "task_surface": "Executable office, retrieval, analytics, SRE and software-engineering workflows",
    "task_count": 106,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": false,
    "repeated_trials": false,
    "grader": "Executable checks plus process judging",
    "data_access": "Public paper, tasks, code and aggregated JSON",
    "conflict": "none",
    "conflict_note": "The 6 harnesses and 8 models are attributed to parties other than Qihoo 360 (checked 18 July 2026). The authors' Qiyuan Tech affiliation could not be conclusively distinguished from Beijing Qiyuan Technology, a 360-linked entity.",
    "weight_tier": "supporting",
    "weight_rationale": "106 executable tasks across 6 harnesses and 8 models, but one trajectory per configuration and a combined score that includes process judging.",
    "source_url": "https://www.harness-bench.ai/",
    "section": "Operational and broad evidence",
    "section_order": 8,
    "overview_summary": "Across the same 8-model pool, average combined score ranged from 52.4 for OpenClaw to 76.2 for NanoBot.",
    "slide_lead": "Harness-Bench ran a 6-harness by 8-model factorial: average combined score ranged from 52.4 to 76.2 across harnesses.",
    "method_summary": "Harness-Bench ran 6 configurable harnesses with 8 models on 106 executable professional tasks, producing 5,088 trajectories. Only 22 tasks are software engineering.",
    "conclusion": "NanoBot had the highest harness-average combined score and completion rate. The size and direction of the harness effect still changed by model.",
    "limitation": "There is one trajectory per task and configuration. Combined score includes process judging, and the public repository currently has no licence file."
  },
  {
    "id": "harvey-lab",
    "name": "Harvey LAB harness ablation",
    "publisher": "Harvey and Applied Compute",
    "published": "2026-06-22",
    "date_basis": "Article publication date",
    "task_surface": "Held-out legal workflows scored against expert-written criteria",
    "task_count": 180,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": false,
    "repeated_trials": false,
    "grader": "GPT-5 Mini against expert criteria, graded 3 times and averaged",
    "data_access": "Public aggregate chart; benchmark tasks and baseline harness are public",
    "conflict": "compared vendor",
    "conflict_note": "Applied Compute, a co-publisher, built the improved harness under test.",
    "weight_tier": "contextual",
    "weight_rationale": "A vendor comparison with an aggregate-only chart, undefined error values and a non-public test split and harness.",
    "source_url": "https://www.harvey.ai/blog/training-a-legal-agent-with-applied-compute",
    "section": "Focused domain evidence",
    "section_order": 14,
    "overview_summary": "The same harness intervention ranged from 0.9 points worse to 23 points better, depending on the model.",
    "slide_lead": "Harvey tested 180 legal tasks: the Applied Compute harness ranged from 0.9 points worse to 23 points better across 6 fixed models.",
    "method_summary": "Harvey compared its baseline LAB-style harness with an Applied Compute harness on 180 held-out legal tasks and about 10,000 criteria. It held 6 models and the grader fixed.",
    "conclusion": "The treatment effect was heterogeneous: slightly negative for GLM 5.1, neutral for Kimi K2.6 and strongly positive for Sonnet 4.6.",
    "limitation": "The exact test split and Applied Compute harness are not public. Published error values are not defined, and agent cost, runtime and tokens are not reported."
  },
  {
    "id": "csi-scaffolds",
    "name": "CSI scaffold comparison",
    "publisher": "Alias Robotics",
    "published": "2026-05-27",
    "date_basis": "arXiv v1 submission date",
    "task_surface": "CyBench capture-the-flag security challenges",
    "task_count": 33,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": false,
    "grader": "Challenge flag verification",
    "data_access": "Public paper and project page; no dedicated experiment artefact found",
    "conflict": "compared vendor",
    "conflict_note": "Both compared scaffolds are Alias Robotics products; the comparison is between its own backends.",
    "weight_tier": "contextual",
    "weight_rationale": "A vendor comparison of 33 challenges with one run per scaffold and no public experiment artefact.",
    "source_url": "https://aliasrobotics.com/scaffolds.php",
    "section": "Focused domain evidence",
    "section_order": 17,
    "overview_summary": "CSI::Claude and CSI::Codex both solved 15 of 33 tasks, but Codex cost one-third as much and finished faster.",
    "slide_lead": "CSI ran 5 scaffolds with one model on 33 security challenges: Claude and Codex tied on solves, while Codex cost one-third as much.",
    "method_summary": "Alias Robotics ran 5 scaffolds with the same private alias2-mini model on 33 CyBench capture-the-flag challenges and recorded solves, total API cost, wall time, commands and tokens.",
    "conclusion": "CSI::Claude and CSI::Codex both solved 15 tasks. CSI::Codex used one-third of the published cost and 31% less wall time.",
    "limitation": "There is one run per scaffold and challenge. The model is proprietary, the task set is small and no dedicated experiment artefact is public."
  },
  {
    "id": "scaffold-effect",
    "name": "The Scaffold Effect",
    "publisher": "Synapse Social",
    "published": "2026-04-29",
    "date_basis": "Paper publication date",
    "task_surface": "A 50-task slice of Terminal-Bench Pro",
    "task_count": 50,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": false,
    "repeated_trials": false,
    "grader": "Terminal-Bench Pro task checks",
    "data_access": "Public paper, raw results, prompts and analysis",
    "conflict": "none",
    "weight_tier": "supporting",
    "weight_rationale": "A public 50-task slice with raw results, but one run per configuration and incomplete token accounting.",
    "source_url": "https://synapsesocial.com/papers/69f1545d879cb923c49448be",
    "section": "Small and diagnostic evidence",
    "section_order": 22,
    "overview_summary": "Goose used about 41 times fewer reported tokens than OpenCode for similar pass rates on 2 fixed models.",
    "slide_lead": "The Scaffold Effect ran 300 trials: Goose used about 41 times fewer reported tokens than OpenCode, with pass rates within 2 to 8 points.",
    "method_summary": "The study ran 2 models through Goose, OpenCode and OpenHands-SDK on a stratified 50-task slice of Terminal-Bench Pro, with identical environments and a 40-turn budget.",
    "conclusion": "Resource use separated the harnesses more clearly than pass rate. OpenCode used about 41 times as many reported tokens as Goose.",
    "limitation": "There is one run per task and configuration. Token accounting differs by harness and is incomplete; cost is missing for 2 harnesses."
  },
  {
    "id": "hal-swe-mini",
    "name": "HAL SWE-bench Verified Mini",
    "publisher": "Princeton HAL",
    "published": "2025-10-13",
    "date_basis": "arXiv v1 submission date",
    "task_surface": "A fixed 50-task subset of SWE-bench Verified",
    "task_count": 50,
    "matched_model_comparison": true,
    "includes_pi": false,
    "reports_cost": true,
    "repeated_trials": false,
    "grader": "SWE-bench repository tests",
    "data_access": "Public leaderboard and downloadable traces",
    "conflict": "none",
    "weight_tier": "supporting",
    "weight_rationale": "An independent leaderboard with downloadable traces, but a 50-task subset with mostly single runs.",
    "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini",
    "section": "Small and diagnostic evidence",
    "section_order": 23,
    "overview_summary": "SWE-Agent usually solved more tasks than HAL Generalist, but often at several times the reported cost.",
    "slide_lead": "HAL compared 15 models on 50 SWE-bench tasks: SWE-Agent usually solved more, while HAL Generalist usually cost less.",
    "method_summary": "Princeton HAL ran the same 50 SWE-bench Verified Mini tasks under SWE-Agent and HAL Generalist for 15 models and effort settings, reporting pass rate and total provider cost.",
    "conclusion": "SWE-Agent led 14 of 15 matched pairs on pass rate. HAL Generalist was often cheaper, showing a strong quality-cost trade-off rather than a universal harness win.",
    "limitation": "Most entries have one evaluation run. SWE-Agent cost excludes caching, and the archived trace repository has no licence file."
  },
  {
    "id": "deepseek-v4-1-flash",
    "name": "DeepSeek V4.1 Flash scaffold evaluation",
    "publisher": "DeepSeek",
    "published": "2026-09-10",
    "date_basis": "Model and technical report release date",
    "task_surface": "Repository issue resolution and containerized command-line tasks",
    "task_count": 202,
    "matched_model_comparison": true,
    "includes_pi": true,
    "reports_cost": false,
    "repeated_trials": true,
    "binomial_interval": false,
    "grader": "DeepSWE repository tests and Terminal-Bench 2.1 final-state tests",
    "data_access": "Public model card, technical report, selected harness code and aggregate tables; no task-level results or trajectories",
    "conflict": "compared vendor",
    "conflict_note": "DeepSeek publishes the model and the three DeepSeek Harness configurations in the comparison.",
    "weight_tier": "supporting",
    "weight_rationale": "A 202-task, repeated, fixed-model scaffold comparison with explicit shared inference controls, published by the vendor of the model and three compared harnesses.",
    "citation_note": "Cite the aggregate scaffold spread as supporting vendor evidence. No task-level results, confidence intervals, complete retry policy or cost data are public.",
    "component_suites": [
      {
        "name": "DeepSWE v1.1",
        "producer": "Datacurve",
        "task_count": 113,
        "provenance": "Human-written issue-resolution tasks on real open-source repositories",
        "overlaps_study": "artificial-analysis"
      },
      {
        "name": "Terminal-Bench 2.1",
        "producer": "Laude Institute",
        "task_count": 89,
        "provenance": "Human-written containerized command-line tasks with final-state checks",
        "overlaps_study": "terminal-bench"
      }
    ],
    "source_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash",
    "section": "Operational and broad evidence",
    "section_order": 12,
    "overview_summary": "With DeepSeek V4.1 Flash fixed, the scaffold spread was 8.7 points on DeepSWE and 6.5 points on Terminal-Bench.",
    "slide_lead": "DeepSeek crossed one model with 8 scaffold configurations: scores spanned 65.5% to 74.2% on DeepSWE and 84.1% to 90.6% on Terminal-Bench.",
    "method_summary": "DeepSeek ran 8 scaffold configurations with the same V4.1 Flash checkpoint, maximum reasoning effort, sampling settings, context limit and round limit. It used 8 samples per DeepSWE task and 3 per Terminal-Bench task.",
    "conclusion": "The observed leader changed by suite. mini-SWE-agent led DeepSWE, while DeepSeek Harness Minimal led Terminal-Bench.",
    "limitation": "DeepSeek publishes the model and three compared configurations. Only aggregate scores are public, and the reproduction package covers a subset of the reported configurations. The Codex score does not map cleanly to 89 tasks times 3 binary samples, so its denominator needs clarification."
  },
  {
    "id": "prime-agent-long-context",
    "name": "Prime Agent long-context comparison",
    "publisher": "Prime Intellect",
    "published": "2026-08-24",
    "date_basis": "Technical report arXiv v1 submission date",
    "task_surface": "Nine long-context reasoning, instruction-following and emulator-construction evaluations",
    "task_count": null,
    "matched_model_comparison": true,
    "includes_pi": true,
    "reports_cost": false,
    "repeated_trials": null,
    "grader": "Benchmark-specific metrics, including accuracy, nDCG@10 and verifier scores",
    "data_access": "Public paper and harness code; aggregate table only for the included comparisons",
    "conflict": "compared vendor",
    "conflict_note": "Prime Intellect builds and publishes Prime Agent, the treatment harness in every included pair.",
    "weight_tier": "supporting",
    "weight_rationale": "A broad fixed-model and fixed-effort matrix across nine evaluations, limited by vendor authorship, missing task counts and no uncertainty intervals.",
    "citation_note": "Use as direction-only supporting evidence. Metrics differ by row, exact task counts and repetitions are mostly unstated, and the paper reports no uncertainty intervals.",
    "source_url": "https://arxiv.org/abs/2608.23552",
    "section": "Focused domain evidence",
    "section_order": 21,
    "overview_summary": "Prime Agent had the higher point estimate in 20 of 27 high-effort pairs, without published uncertainty.",
    "slide_lead": "Prime Intellect compared 3 models at high effort across 9 long-context evaluations: Prime Agent led 20 of 27 point estimates.",
    "method_summary": "Prime Intellect compared Prime Agent with Pi-mono, Claude Code and Codex. It held each nominal model and the high reasoning label fixed, and placed the initial context in a readable file for every harness.",
    "conclusion": "Prime Agent led 8 of 9 GLM-5.2 rows and 6 of 9 rows for each of Opus 5 and GPT-5.6 Sol. The size and direction of the gap changed by model and task.",
    "limitation": "The table gives no uncertainty intervals and mostly omits exact task counts, repetitions, model snapshots and budgets. Prime's launch article reports 0.500 for the Codex OOLONG Yahoo result, while the later paper reports 0.900; this review uses the paper."
  },
  {
    "id": "browser-use-pi-revisions",
    "name": "Browser Use Pi concurrent revision comparison",
    "publisher": "Browser Use",
    "published": "2026-09-09",
    "date_basis": "Public repository release date for the benchmark evidence",
    "task_surface": "Live browser research tasks and an internal hard browser benchmark",
    "task_count": 166,
    "matched_model_comparison": true,
    "includes_pi": true,
    "reports_cost": true,
    "repeated_trials": false,
    "grader": "Luna rubric judgments and Laith binary judgments using the same model as the agent",
    "data_access": "Public aggregate reports, configuration manifests, task-level scores, costs and selected trace audits; benchmark tasks and most raw traces need organization access",
    "conflict": "compared vendor",
    "conflict_note": "Browser Use publishes Browser Use Pi and both runtime revisions under comparison.",
    "weight_tier": "supporting",
    "weight_rationale": "A concurrent matched revision comparison over 166 assigned tasks with public task-level evidence, published by the system vendor and developed on the benchmark tasks.",
    "citation_note": "Cite only the frozen concurrent revisions. The current released helper surface has no evaluation, and historical BrowserCode and Python harness results are nonconcurrent context.",
    "source_url": "https://github.com/browser-use/browser-use-pi/blob/416b9906f0dbb5784ea4aa162345453842c260c3/docs/benchmark-overview.md",
    "section": "Operational and broad evidence",
    "section_order": 13,
    "overview_summary": "A bundled runtime revision improved BU Bench V2 by 7.53 points, but tied on Hard and failed the study's 2-benchmark confirmation rule.",
    "slide_lead": "Browser Use ran concurrent revision comparisons: the candidate gained 7.53 points on 60 Luna tasks and tied at 84 of 106 on Hard.",
    "method_summary": "Browser Use dispatched frozen candidate and reference revisions concurrently with matched model, effort, tasks, judge, browser controls and budgets. The treatment bundled several runtime fixes.",
    "conclusion": "The candidate scored 61.75 versus 54.22 on BU Bench V2. Its paired 95% bootstrap interval was +0.10 to +15.42 points. Both revisions passed 84 of 106 Hard tasks, and the reference lacked one actual judgment.",
    "limitation": "The tasks had been inspected during development, the treatment bundles several changes, and task bootstrapping does not cover judge or provider sampling. The Hard pair failed full judgment coverage, and the current release is not the evaluated candidate."
  }
]
