{
 "contestants": [
  {
   "model": "openai/gpt-6.1-sol",
   "router": false,
   "reps": 1,
   "score_mean": 16,
   "score_min": 16,
   "score_max": 16,
   "n_tasks": 16,
   "cost_per_pass": 0.0748,
   "latency_median": 10.0,
   "tier": {
    "easy": "4/4",
    "medium": "4/4",
    "hard": "4/4",
    "extreme": "4/4"
   },
   "mix": {
    "easy": {
     "gpt-6.1-sol": 4
    },
    "medium": {
     "gpt-6.1-sol": 4
    },
    "hard": {
     "gpt-6.1-sol": 4
    },
    "extreme": {
     "gpt-6.1-sol": 4
    }
   },
   "fails": [],
   "errors": 0
  },
  {
   "model": "anthropic/claude-opus-5.5",
   "router": false,
   "reps": 1,
   "score_mean": 16,
   "score_min": 16,
   "score_max": 16,
   "n_tasks": 16,
   "cost_per_pass": 0.3108,
   "latency_median": 9.2,
   "tier": {
    "easy": "4/4",
    "medium": "4/4",
    "hard": "4/4",
    "extreme": "4/4"
   },
   "mix": {
    "easy": {
     "claude-opus-5.5": 4
    },
    "medium": {
     "claude-opus-5.5": 4
    },
    "hard": {
     "claude-opus-5.5": 4
    },
    "extreme": {
     "claude-opus-5.5": 4
    }
   },
   "fails": [],
   "errors": 0
  },
  {
   "model": "unbiased/pareto",
   "router": true,
   "reps": 3,
   "score_mean": 15.67,
   "score_min": 15,
   "score_max": 16,
   "n_tasks": 16,
   "cost_per_pass": 0.0636,
   "latency_median": 26.7,
   "tier": {
    "easy": "12/12",
    "medium": "12/12",
    "hard": "11/12",
    "extreme": "12/12"
   },
   "mix": {
    "easy": {
     "pareto": 12
    },
    "medium": {
     "pareto": 12
    },
    "hard": {
     "pareto": 12
    },
    "extreme": {
     "pareto": 12
    }
   },
   "fails": [
    "H2"
   ],
   "errors": 0
  },
  {
   "model": "nvidia/switchyard",
   "router": true,
   "reps": 3,
   "score_mean": 15.67,
   "score_min": 15,
   "score_max": 16,
   "n_tasks": 16,
   "cost_per_pass": 0.8334,
   "latency_median": 28.4,
   "tier": {
    "easy": "12/12",
    "medium": "12/12",
    "hard": "11/12",
    "extreme": "12/12"
   },
   "mix": {
    "easy": {
     "deepseek-v4.1-flash": 8,
     "kimi-k3": 4
    },
    "medium": {
     "kimi-k3": 11,
     "deepseek-v4.1-flash": 1
    },
    "hard": {
     "kimi-k3": 12
    },
    "extreme": {
     "kimi-k3": 12
    }
   },
   "fails": [
    "H3"
   ],
   "errors": 0
  },
  {
   "model": "openrouter/auto",
   "router": true,
   "reps": 3,
   "score_mean": 15.33,
   "score_min": 15,
   "score_max": 16,
   "n_tasks": 16,
   "cost_per_pass": 0.0791,
   "latency_median": 9.9,
   "tier": {
    "easy": "12/12",
    "medium": "12/12",
    "hard": "12/12",
    "extreme": "10/12"
   },
   "mix": {
    "easy": {
     "deepseek-v4-flash-0731": 6,
     "deepseek-v4.1-flash": 3,
     "gpt-6-luna": 3
    },
    "medium": {
     "deepseek-v4.1-flash": 12
    },
    "hard": {
     "deepseek-v4.1-flash": 12
    },
    "extreme": {
     "deepseek-v4.1-flash": 12
    }
   },
   "fails": [
    "X3"
   ],
   "errors": 0
  },
  {
   "model": "typesafe/jev-router",
   "router": true,
   "reps": 3,
   "score_mean": 15.33,
   "score_min": 15,
   "score_max": 16,
   "n_tasks": 16,
   "cost_per_pass": 0.1493,
   "latency_median": 11.6,
   "tier": {
    "easy": "12/12",
    "medium": "12/12",
    "hard": "11/12",
    "extreme": "11/12"
   },
   "mix": {
    "easy": {
     "gpt-6-luna": 6,
     "deepseek-v4.1-flash": 3,
     "gemini-3.8-flash": 3
    },
    "medium": {
     "gemini-3.8-flash": 9,
     "deepseek-v4.1-flash": 3
    },
    "hard": {
     "claude-sonnet-5.5": 9,
     "gpt-6.1-sol": 3
    },
    "extreme": {
     "gpt-6.1-sol": 6,
     "claude-sonnet-5.5": 6
    }
   },
   "fails": [
    "H1",
    "X1"
   ],
   "errors": 0
  },
  {
   "model": "deepseek/deepseek-v4.1-flash",
   "router": false,
   "reps": 1,
   "score_mean": 15,
   "score_min": 15,
   "score_max": 15,
   "n_tasks": 16,
   "cost_per_pass": 0.0397,
   "latency_median": 22.4,
   "tier": {
    "easy": "4/4",
    "medium": "4/4",
    "hard": "4/4",
    "extreme": "3/4"
   },
   "mix": {
    "easy": {
     "deepseek-v4.1-flash": 4
    },
    "medium": {
     "deepseek-v4.1-flash": 4
    },
    "hard": {
     "deepseek-v4.1-flash": 4
    },
    "extreme": {
     "deepseek-v4.1-flash": 4
    }
   },
   "fails": [
    "X1"
   ],
   "errors": 0
  },
  {
   "model": "google/gemini-3.8-flash",
   "router": false,
   "reps": 1,
   "score_mean": 15,
   "score_min": 15,
   "score_max": 15,
   "n_tasks": 16,
   "cost_per_pass": 0.1484,
   "latency_median": 13.5,
   "tier": {
    "easy": "4/4",
    "medium": "4/4",
    "hard": "3/4",
    "extreme": "4/4"
   },
   "mix": {
    "easy": {
     "gemini-3.8-flash": 4
    },
    "medium": {
     "gemini-3.8-flash": 4
    },
    "hard": {
     "gemini-3.8-flash": 4
    },
    "extreme": {
     "gemini-3.8-flash": 4
    }
   },
   "fails": [
    "H2"
   ],
   "errors": 0
  }
 ],
 "task_pass": {
  "E1": "16/16",
  "E2": "16/16",
  "E3": "16/16",
  "E4": "16/16",
  "M1": "16/16",
  "H1": "15/16",
  "M2": "16/16",
  "M3": "16/16",
  "M4": "16/16",
  "H4": "16/16",
  "H2": "14/16",
  "H3": "15/16",
  "X1": "14/16",
  "X2": "16/16",
  "X3": "14/16",
  "X4": "16/16"
 },
 "total_cost": 3.95,
 "calls": 256
}