{
 "name": "VulcanBench Eval Suite 3 leaderboard",
 "suite": "v3",
 "url": "https://vulcanbench.com/leaderboard.html",
 "updated": "2026-08-25",
 "tasks": 23,
 "models": 15,
 "columns_total": 42,
 "runs_total": 1651,
 "license": "https://github.com/morganlinton/VulcanBench/blob/main/LICENSE",
 "notes": [
  "pass_at_1_pct is the mean per-task success rate for the column. Values with one decimal come from the run data or the numbered reports; whole numbers are as rendered on the 2026-08-13 board card.",
  "se_pct (\u00b11 standard error) is published for each model's best column; per-column stderr for the rest lives in the run data.",
  "usd_per_task_run is total list-price API spend for the column divided by runs. Negotiated, cached-input, and batch rates differ.",
  "min_per_task is average sandbox wall-clock, except Qwen3.8-27B, GLM 5.3, and Muse Spark 1.2 columns, which report medians (Reports 17, 18, and 19).",
  "effort_is_provider_default is true where the provider documents that level as the unset-request default, false where a different level is documented as default, and null where no default is documented in our reports.",
  "task_coverage below 23/23 marks partial coverage: safety-filter refusals (Claude Fable 5), provider-side failures, or an incomplete fresh sweep.",
  "A model's best column is the effort level that scored highest for it; ties break to the cheaper run, and a partial-coverage sweep cannot outrank a full one.",
  "GLM 5.3 columns are its raw z.ai API (Report 18). The same report's ZCode-harness results (87.0% at max) measure model plus product and are excluded from the board.",
  "Muse Spark 1.2 columns are the Meta Model API (Report 19), one attempt per level; Meta documents no unset-request default, so effort_is_provider_default is null for all three."
 ],
 "columns": [
  {
   "rank": 1,
   "model": "Grok 4.5",
   "lab": "xAI",
   "effort": "high",
   "effort_is_provider_default": null,
   "best_for_model": true,
   "pass_at_1_pct": 89.9,
   "se_pct": 6.1,
   "usd_per_task_run": 0.47,
   "min_per_task": 8.4,
   "runs": 50,
   "task_coverage": "23/23"
  },
  {
   "rank": 2,
   "model": "Claude Fable 5",
   "lab": "Anthropic",
   "effort": "low",
   "effort_is_provider_default": null,
   "best_for_model": true,
   "pass_at_1_pct": 89.5,
   "se_pct": 7.2,
   "usd_per_task_run": 0.51,
   "min_per_task": 3.5,
   "runs": 19,
   "task_coverage": "19/23"
  },
  {
   "rank": 3,
   "model": "DeepSeek V4-Flash",
   "lab": "DeepSeek",
   "effort": "max",
   "effort_is_provider_default": null,
   "best_for_model": true,
   "pass_at_1_pct": 88.4,
   "se_pct": 6.2,
   "usd_per_task_run": 0.06,
   "min_per_task": 11.2,
   "runs": 68,
   "task_coverage": "23/23"
  },
  {
   "rank": 4,
   "model": "DeepSeek V4-Flash",
   "lab": "DeepSeek",
   "effort": "high",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 87.0,
   "se_pct": null,
   "usd_per_task_run": 0.08,
   "min_per_task": 12.0,
   "runs": 69,
   "task_coverage": "23/23"
  },
  {
   "rank": 5,
   "model": "DeepSeek V4 Pro",
   "lab": "DeepSeek",
   "effort": "high",
   "effort_is_provider_default": null,
   "best_for_model": true,
   "pass_at_1_pct": 87.0,
   "se_pct": 6.2,
   "usd_per_task_run": 0.11,
   "min_per_task": 7.4,
   "runs": 69,
   "task_coverage": "23/23"
  },
  {
   "rank": 6,
   "model": "GPT-5.6 Terra",
   "lab": "OpenAI",
   "effort": "medium",
   "effort_is_provider_default": null,
   "best_for_model": true,
   "pass_at_1_pct": 87.0,
   "se_pct": 7.2,
   "usd_per_task_run": 0.19,
   "min_per_task": 2.3,
   "runs": 69,
   "task_coverage": "23/23"
  },
  {
   "rank": 7,
   "model": "Muse Spark 1.2",
   "lab": "Meta",
   "effort": "low",
   "effort_is_provider_default": null,
   "best_for_model": true,
   "pass_at_1_pct": 87.0,
   "se_pct": 7.0,
   "usd_per_task_run": 0.36,
   "min_per_task": 3.2,
   "runs": 23,
   "task_coverage": "23/23"
  },
  {
   "rank": 8,
   "model": "Claude Opus 5",
   "lab": "Anthropic",
   "effort": "low",
   "effort_is_provider_default": null,
   "best_for_model": true,
   "pass_at_1_pct": 87.0,
   "se_pct": 7.2,
   "usd_per_task_run": 0.61,
   "min_per_task": 5.2,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/10-opus5-effort.html"
  },
  {
   "rank": 9,
   "model": "Grok 4.6",
   "lab": "xAI",
   "effort": "medium",
   "effort_is_provider_default": false,
   "best_for_model": true,
   "pass_at_1_pct": 87.0,
   "se_pct": 7.2,
   "usd_per_task_run": 0.69,
   "min_per_task": 14.2,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/14-grok-46-effort.html"
  },
  {
   "rank": 10,
   "model": "GPT-5.6 Sol",
   "lab": "OpenAI",
   "effort": "high",
   "effort_is_provider_default": null,
   "best_for_model": true,
   "pass_at_1_pct": 87.0,
   "se_pct": 7.2,
   "usd_per_task_run": 0.69,
   "min_per_task": 4.2,
   "runs": 23,
   "task_coverage": "23/23"
  },
  {
   "rank": 11,
   "model": "Grok 4.5",
   "lab": "xAI",
   "effort": "low",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 86,
   "se_pct": null,
   "usd_per_task_run": 0.11,
   "min_per_task": 1.7,
   "runs": 21,
   "task_coverage": "21/23"
  },
  {
   "rank": 12,
   "model": "GPT-5.6 Luna",
   "lab": "OpenAI",
   "effort": "max",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 86,
   "se_pct": null,
   "usd_per_task_run": 0.84,
   "min_per_task": 13,
   "runs": 21,
   "task_coverage": "21/23"
  },
  {
   "rank": 13,
   "model": "GPT-5.6 Terra",
   "lab": "OpenAI",
   "effort": "max",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 86,
   "se_pct": null,
   "usd_per_task_run": 2.29,
   "min_per_task": 14,
   "runs": 21,
   "task_coverage": "21/23"
  },
  {
   "rank": 14,
   "model": "DeepSeek V4-Flash",
   "lab": "DeepSeek",
   "effort": "low",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 86,
   "se_pct": null,
   "usd_per_task_run": 0.04,
   "min_per_task": 8.4,
   "runs": 69,
   "task_coverage": "23/23"
  },
  {
   "rank": 15,
   "model": "GPT-5.6 Luna",
   "lab": "OpenAI",
   "effort": "high",
   "effort_is_provider_default": null,
   "best_for_model": true,
   "pass_at_1_pct": 85.5,
   "se_pct": 6.6,
   "usd_per_task_run": 0.17,
   "min_per_task": 3.8,
   "runs": 69,
   "task_coverage": "23/23"
  },
  {
   "rank": 16,
   "model": "Grok 4.5",
   "lab": "xAI",
   "effort": "medium",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 86,
   "se_pct": null,
   "usd_per_task_run": 0.43,
   "min_per_task": 8.5,
   "runs": 50,
   "task_coverage": "23/23"
  },
  {
   "rank": 17,
   "model": "GPT-5.6 Terra",
   "lab": "OpenAI",
   "effort": "high",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 86,
   "se_pct": null,
   "usd_per_task_run": 0.38,
   "min_per_task": 4.2,
   "runs": 69,
   "task_coverage": "23/23"
  },
  {
   "rank": 18,
   "model": "Claude Fable 5",
   "lab": "Anthropic",
   "effort": "high",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 85,
   "se_pct": null,
   "usd_per_task_run": 0.9,
   "min_per_task": 4.5,
   "runs": 20,
   "task_coverage": "20/23"
  },
  {
   "rank": 19,
   "model": "Grok 4.6",
   "lab": "xAI",
   "effort": "low",
   "effort_is_provider_default": false,
   "best_for_model": false,
   "pass_at_1_pct": 82.6,
   "se_pct": null,
   "usd_per_task_run": 0.18,
   "min_per_task": 4.8,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/14-grok-46-effort.html"
  },
  {
   "rank": 20,
   "model": "GPT-5.6 Sol",
   "lab": "OpenAI",
   "effort": "medium",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 82.6,
   "se_pct": null,
   "usd_per_task_run": 0.38,
   "min_per_task": 2.8,
   "runs": 23,
   "task_coverage": "23/23"
  },
  {
   "rank": 21,
   "model": "Qwen3.8-27B",
   "lab": "Alibaba",
   "effort": "medium",
   "effort_is_provider_default": false,
   "best_for_model": true,
   "pass_at_1_pct": 82.6,
   "se_pct": 8.1,
   "usd_per_task_run": 0.48,
   "min_per_task": 6.8,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/17-qwen38-27b-effort.html"
  },
  {
   "rank": 22,
   "model": "Claude Opus 5",
   "lab": "Anthropic",
   "effort": "medium",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 82.6,
   "se_pct": null,
   "usd_per_task_run": 1.05,
   "min_per_task": 7.6,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/10-opus5-effort.html"
  },
  {
   "rank": 23,
   "model": "GPT-5.6 Terra",
   "lab": "OpenAI",
   "effort": "low",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 81,
   "se_pct": null,
   "usd_per_task_run": 0.15,
   "min_per_task": 2.0,
   "runs": 69,
   "task_coverage": "23/23"
  },
  {
   "rank": 24,
   "model": "Qwen3.8-Max",
   "lab": "Alibaba",
   "effort": "low",
   "effort_is_provider_default": false,
   "best_for_model": true,
   "pass_at_1_pct": 81.2,
   "se_pct": 7.8,
   "usd_per_task_run": 0.61,
   "min_per_task": 19.6,
   "runs": 69,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/12-qwen38-max.html"
  },
  {
   "rank": 25,
   "model": "Claude Fable 5",
   "lab": "Anthropic",
   "effort": "medium",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 81,
   "se_pct": null,
   "usd_per_task_run": 0.7,
   "min_per_task": 3.8,
   "runs": 21,
   "task_coverage": "21/23"
  },
  {
   "rank": 26,
   "model": "GPT-5.6 Luna",
   "lab": "OpenAI",
   "effort": "medium",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 80,
   "se_pct": null,
   "usd_per_task_run": 0.06,
   "min_per_task": 2.1,
   "runs": 69,
   "task_coverage": "23/23"
  },
  {
   "rank": 27,
   "model": "GPT-5.6 Sol",
   "lab": "OpenAI",
   "effort": "low",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 78.3,
   "se_pct": null,
   "usd_per_task_run": 0.17,
   "min_per_task": 1.5,
   "runs": 23,
   "task_coverage": "23/23"
  },
  {
   "rank": 28,
   "model": "GLM 5.3",
   "lab": "Z.ai",
   "effort": "low",
   "effort_is_provider_default": false,
   "best_for_model": true,
   "pass_at_1_pct": 78.3,
   "se_pct": 8.6,
   "usd_per_task_run": 0.32,
   "min_per_task": 3.1,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/18-glm53-zcode-harness.html"
  },
  {
   "rank": 29,
   "model": "Grok 4.6",
   "lab": "xAI",
   "effort": "xhigh",
   "effort_is_provider_default": false,
   "best_for_model": false,
   "pass_at_1_pct": 78.3,
   "se_pct": null,
   "usd_per_task_run": 0.88,
   "min_per_task": 15.8,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/14-grok-46-effort.html"
  },
  {
   "rank": 30,
   "model": "Claude Opus 5",
   "lab": "Anthropic",
   "effort": "high",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 78.3,
   "se_pct": null,
   "usd_per_task_run": 1.9,
   "min_per_task": 13.6,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/10-opus5-effort.html"
  },
  {
   "rank": 31,
   "model": "GPT-5.6 Luna",
   "lab": "OpenAI",
   "effort": "low",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 77,
   "se_pct": null,
   "usd_per_task_run": 0.03,
   "min_per_task": 1.2,
   "runs": 69,
   "task_coverage": "23/23"
  },
  {
   "rank": 32,
   "model": "Qwen3.8-27B",
   "lab": "Alibaba",
   "effort": "low",
   "effort_is_provider_default": false,
   "best_for_model": false,
   "pass_at_1_pct": 76.8,
   "se_pct": 8.2,
   "usd_per_task_run": 0.71,
   "min_per_task": 7.1,
   "runs": 69,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/17-qwen38-27b-effort.html"
  },
  {
   "rank": 33,
   "model": "Claude Haiku 4.5",
   "lab": "Anthropic",
   "effort": "default",
   "effort_is_provider_default": true,
   "best_for_model": true,
   "pass_at_1_pct": 76.2,
   "se_pct": 9.5,
   "usd_per_task_run": 0.42,
   "min_per_task": 5.3,
   "runs": 21,
   "task_coverage": "21/23"
  },
  {
   "rank": 34,
   "model": "GLM 5.3",
   "lab": "Z.ai",
   "effort": "high",
   "effort_is_provider_default": false,
   "best_for_model": false,
   "pass_at_1_pct": 73.9,
   "se_pct": null,
   "usd_per_task_run": 0.45,
   "min_per_task": 9.4,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/18-glm53-zcode-harness.html"
  },
  {
   "rank": 35,
   "model": "Grok 4.6",
   "lab": "xAI",
   "effort": "high",
   "effort_is_provider_default": true,
   "best_for_model": false,
   "pass_at_1_pct": 73.9,
   "se_pct": null,
   "usd_per_task_run": 0.55,
   "min_per_task": 16.3,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/14-grok-46-effort.html"
  },
  {
   "rank": 36,
   "model": "Qwen3.8-27B",
   "lab": "Alibaba",
   "effort": "xhigh",
   "effort_is_provider_default": true,
   "best_for_model": false,
   "pass_at_1_pct": 73.9,
   "se_pct": 9.4,
   "usd_per_task_run": 0.69,
   "min_per_task": 17.1,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/17-qwen38-27b-effort.html"
  },
  {
   "rank": 37,
   "model": "Kimi K3",
   "lab": "Moonshot",
   "effort": "extra-high",
   "effort_is_provider_default": null,
   "best_for_model": true,
   "pass_at_1_pct": 73.7,
   "se_pct": 10.4,
   "usd_per_task_run": 0.69,
   "min_per_task": 17.2,
   "runs": 19,
   "task_coverage": "19/23"
  },
  {
   "rank": 38,
   "model": "Muse Spark 1.2",
   "lab": "Meta",
   "effort": "high",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 73.9,
   "se_pct": 9.2,
   "usd_per_task_run": 0.91,
   "min_per_task": 12.4,
   "runs": 23,
   "task_coverage": "23/23"
  },
  {
   "rank": 39,
   "model": "Qwen3.8-Max",
   "lab": "Alibaba",
   "effort": "medium",
   "effort_is_provider_default": false,
   "best_for_model": false,
   "pass_at_1_pct": 71.0,
   "se_pct": null,
   "usd_per_task_run": 0.56,
   "min_per_task": 21.7,
   "runs": 69,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/12-qwen38-max.html"
  },
  {
   "rank": 40,
   "model": "GLM 5.3",
   "lab": "Z.ai",
   "effort": "max",
   "effort_is_provider_default": true,
   "best_for_model": false,
   "pass_at_1_pct": 65.2,
   "se_pct": null,
   "usd_per_task_run": 0.78,
   "min_per_task": 20.1,
   "runs": 23,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/18-glm53-zcode-harness.html"
  },
  {
   "rank": 41,
   "model": "Qwen3.8-Max",
   "lab": "Alibaba",
   "effort": "xhigh",
   "effort_is_provider_default": true,
   "best_for_model": false,
   "pass_at_1_pct": 55.1,
   "se_pct": null,
   "usd_per_task_run": 0.71,
   "min_per_task": 25.5,
   "runs": 64,
   "task_coverage": "23/23",
   "source": "https://vulcanbench.com/benchmarks/12-qwen38-max.html"
  },
  {
   "rank": 42,
   "model": "Muse Spark 1.2",
   "lab": "Meta",
   "effort": "xhigh",
   "effort_is_provider_default": null,
   "best_for_model": false,
   "pass_at_1_pct": 52.2,
   "se_pct": 10.4,
   "usd_per_task_run": 1.18,
   "min_per_task": 20.1,
   "runs": 23,
   "task_coverage": "23/23"
  }
 ]
}