{
  "edition": "2026-w43",
  "as_of": "2026-10-21T23:00:00Z",
  "method_id": "pickbench_bench_key_cluster_bootstrap_v1",
  "ci_level": 0.95,
  "ci_method": "key_cluster_percentile_bootstrap",
  "n_boot": 2000,
  "seed": 20261020,
  "replicates_r": 3,
  "prereg_hash": "8b23184a8bf8ed28e8b4e94fe1af8a3e9721cde4143c619d8683f00342a847c8",
  "sample_data": false,
  "conditions": {
    "mode": "closed_book",
    "web_search": false,
    "gateway": "openrouter",
    "pinned_providers": true,
    "panel_id": "api-live-pickbench-fast-2026-10-20-v5--api-live-pickben-68273e87",
    "prompt_version": "pickbench-prompt-v3",
    "as_of": "2026-10-21T22:01:25Z",
    "panel_sha256": "02a52f4255278cc0eb76e45bc745f0e5b2d45a895f865734fb0e2e1f2fd8649d"
  },
  "minimums": {
    "min_tasks": 100,
    "min_tailoring_pairs": 100,
    "min_decided_votes": 300,
    "min_vote_keys": 40,
    "min_coarse_netfams": 30
  },
  "order_by": "model_label_alphabetical",
  "people_score_state": "collecting_votes",
  "people_score_owner_decision_id": null,
  "tasks_total": 750,
  "models": [
    {
      "judge_id": "jdg-live-claude-haiku-5-5-20261007-or-v1",
      "model_id": "anthropic/claude-haiku-5.5-20261007",
      "model_label": "anthropic/claude-haiku-5.5-20261007 @ claude-haiku-5.5-20261007 via OpenRouter",
      "gateway": "openrouter",
      "people_score": {
        "state": "collecting_votes",
        "tier": null,
        "position": null,
        "rating": null,
        "ci_low": null,
        "ci_high": null,
        "decided_votes": 0,
        "keys_n": 0,
        "coarse_netfams_n": 0
      },
      "answer_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "ok_tasks": 744,
        "infra_failures": 0,
        "status_breakdown": {
          "none": 0,
          "refusal": 0,
          "invalid": 6,
          "truncated": 0,
          "error": 0,
          "provider_mismatch": 0
        },
        "estimate": 0.992,
        "ci_low": 0.984,
        "ci_high": 0.997333
      },
      "reference_list_rate": {
        "state": "published",
        "eligible_tasks": 744,
        "matched_brands": 2019,
        "named_brands": 2192,
        "unmatched_brands": 173,
        "estimate": 0.921077,
        "ci_low": 0.910427,
        "ci_high": 0.931303
      },
      "consistency": {
        "state": "published",
        "replicates_r": 3,
        "eligible_tasks": 737,
        "consistent_tasks": 599,
        "excluded_incomplete_tasks": 13,
        "estimate": 0.812754,
        "ci_low": 0.782904,
        "ci_high": 0.841248,
        "pairwise_agreement": 0.86929,
        "pairwise_ci_low": 0.848033,
        "pairwise_ci_high": 0.888738
      },
      "tailoring": {
        "state": "published",
        "eligible_pairs": 1234,
        "differing_pairs": 645,
        "clusters_n": 125,
        "estimate": 0.52269,
        "ci_low": 0.466986,
        "ci_high": 0.578105
      },
      "served_provider": "anthropic",
      "served_providers": [
        {
          "provider": "anthropic",
          "answers": 2250
        }
      ]
    },
    {
      "judge_id": "jdg-live-claude-opus-5-5-20260921-or-v1",
      "model_id": "anthropic/claude-opus-5.5-20260921",
      "model_label": "anthropic/claude-opus-5.5-20260921 @ claude-opus-5.5-20260921 via OpenRouter",
      "gateway": "openrouter",
      "people_score": {
        "state": "collecting_votes",
        "tier": null,
        "position": null,
        "rating": null,
        "ci_low": null,
        "ci_high": null,
        "decided_votes": 0,
        "keys_n": 0,
        "coarse_netfams_n": 0
      },
      "answer_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "ok_tasks": 724,
        "infra_failures": 0,
        "status_breakdown": {
          "none": 24,
          "refusal": 0,
          "invalid": 2,
          "truncated": 0,
          "error": 0,
          "provider_mismatch": 0
        },
        "estimate": 0.965333,
        "ci_low": 0.950667,
        "ci_high": 0.978667
      },
      "reference_list_rate": {
        "state": "published",
        "eligible_tasks": 724,
        "matched_brands": 1875,
        "named_brands": 2103,
        "unmatched_brands": 228,
        "estimate": 0.891583,
        "ci_low": 0.879769,
        "ci_high": 0.903185
      },
      "consistency": {
        "state": "published",
        "replicates_r": 3,
        "eligible_tasks": 703,
        "consistent_tasks": 622,
        "excluded_incomplete_tasks": 47,
        "estimate": 0.88478,
        "ci_low": 0.860597,
        "ci_high": 0.907539,
        "pairwise_agreement": 0.920816,
        "pairwise_ci_low": 0.904682,
        "pairwise_ci_high": 0.935989
      },
      "tailoring": {
        "state": "published",
        "eligible_pairs": 1176,
        "differing_pairs": 727,
        "clusters_n": 125,
        "estimate": 0.618197,
        "ci_low": 0.565767,
        "ci_high": 0.667837
      },
      "served_provider": "anthropic",
      "served_providers": [
        {
          "provider": "anthropic",
          "answers": 2250
        }
      ]
    },
    {
      "judge_id": "jdg-live-claude-sonnet-5-5-20260928-or-v1",
      "model_id": "anthropic/claude-sonnet-5.5-20260928",
      "model_label": "anthropic/claude-sonnet-5.5-20260928 @ claude-sonnet-5.5-20260928 via OpenRouter",
      "gateway": "openrouter",
      "people_score": {
        "state": "collecting_votes",
        "tier": null,
        "position": null,
        "rating": null,
        "ci_low": null,
        "ci_high": null,
        "decided_votes": 0,
        "keys_n": 0,
        "coarse_netfams_n": 0
      },
      "answer_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "ok_tasks": 736,
        "infra_failures": 0,
        "status_breakdown": {
          "none": 6,
          "refusal": 0,
          "invalid": 8,
          "truncated": 0,
          "error": 0,
          "provider_mismatch": 0
        },
        "estimate": 0.981333,
        "ci_low": 0.970667,
        "ci_high": 0.990667
      },
      "reference_list_rate": {
        "state": "published",
        "eligible_tasks": 736,
        "matched_brands": 1923,
        "named_brands": 2130,
        "unmatched_brands": 207,
        "estimate": 0.902817,
        "ci_low": 0.891422,
        "ci_high": 0.91429
      },
      "consistency": {
        "state": "published",
        "replicates_r": 3,
        "eligible_tasks": 723,
        "consistent_tasks": 602,
        "excluded_incomplete_tasks": 27,
        "estimate": 0.832642,
        "ci_low": 0.803596,
        "ci_high": 0.858921,
        "pairwise_agreement": 0.887506,
        "pairwise_ci_low": 0.868603,
        "pairwise_ci_high": 0.905486
      },
      "tailoring": {
        "state": "published",
        "eligible_pairs": 1198,
        "differing_pairs": 622,
        "clusters_n": 125,
        "estimate": 0.519199,
        "ci_low": 0.461773,
        "ci_high": 0.574384
      },
      "served_provider": "anthropic",
      "served_providers": [
        {
          "provider": "anthropic",
          "answers": 2250
        }
      ]
    },
    {
      "judge_id": "jdg-live-deepseek-v4-pro-20260813-or-v1",
      "model_id": "deepseek/deepseek-v4-pro-20260813",
      "model_label": "deepseek/deepseek-v4-pro-20260813 @ deepseek-v4-pro-20260813 via OpenRouter",
      "gateway": "openrouter",
      "people_score": {
        "state": "collecting_votes",
        "tier": null,
        "position": null,
        "rating": null,
        "ci_low": null,
        "ci_high": null,
        "decided_votes": 0,
        "keys_n": 0,
        "coarse_netfams_n": 0
      },
      "answer_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "ok_tasks": 750,
        "infra_failures": 0,
        "status_breakdown": {
          "none": 0,
          "refusal": 0,
          "invalid": 0,
          "truncated": 0,
          "error": 0,
          "provider_mismatch": 0
        },
        "estimate": 1,
        "ci_low": 1,
        "ci_high": 1
      },
      "reference_list_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "matched_brands": 1908,
        "named_brands": 2155,
        "unmatched_brands": 247,
        "estimate": 0.885383,
        "ci_low": 0.872846,
        "ci_high": 0.897875
      },
      "consistency": {
        "state": "published",
        "replicates_r": 3,
        "eligible_tasks": 750,
        "consistent_tasks": 680,
        "excluded_incomplete_tasks": 0,
        "estimate": 0.906667,
        "ci_low": 0.884,
        "ci_high": 0.928,
        "pairwise_agreement": 0.937333,
        "pairwise_ci_low": 0.922222,
        "pairwise_ci_high": 0.951111
      },
      "tailoring": {
        "state": "published",
        "eligible_pairs": 1250,
        "differing_pairs": 719,
        "clusters_n": 125,
        "estimate": 0.5752,
        "ci_low": 0.5184,
        "ci_high": 0.62882
      },
      "served_provider": "deepinfra",
      "served_providers": [
        {
          "provider": "deepinfra",
          "answers": 2250
        }
      ]
    },
    {
      "judge_id": "jdg-live-deepseek-v4-1-flash-20260910-or-v2",
      "model_id": "deepseek/deepseek-v4.1-flash-20260910",
      "model_label": "deepseek/deepseek-v4.1-flash-20260910 @ deepseek-v4.1-flash-20260910 via OpenRouter",
      "gateway": "openrouter",
      "people_score": {
        "state": "collecting_votes",
        "tier": null,
        "position": null,
        "rating": null,
        "ci_low": null,
        "ci_high": null,
        "decided_votes": 0,
        "keys_n": 0,
        "coarse_netfams_n": 0
      },
      "answer_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "ok_tasks": 750,
        "infra_failures": 0,
        "status_breakdown": {
          "none": 0,
          "refusal": 0,
          "invalid": 0,
          "truncated": 0,
          "error": 0,
          "provider_mismatch": 0
        },
        "estimate": 1,
        "ci_low": 1,
        "ci_high": 1
      },
      "reference_list_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "matched_brands": 1932,
        "named_brands": 2165,
        "unmatched_brands": 233,
        "estimate": 0.892379,
        "ci_low": 0.880017,
        "ci_high": 0.903864
      },
      "consistency": {
        "state": "published",
        "replicates_r": 3,
        "eligible_tasks": 750,
        "consistent_tasks": 666,
        "excluded_incomplete_tasks": 0,
        "estimate": 0.888,
        "ci_low": 0.865333,
        "ci_high": 0.909333,
        "pairwise_agreement": 0.924889,
        "pairwise_ci_low": 0.910211,
        "pairwise_ci_high": 0.939111
      },
      "tailoring": {
        "state": "published",
        "eligible_pairs": 1250,
        "differing_pairs": 758,
        "clusters_n": 125,
        "estimate": 0.6064,
        "ci_low": 0.55518,
        "ci_high": 0.652
      },
      "served_provider": "deepinfra",
      "served_providers": [
        {
          "provider": "deepinfra",
          "answers": 2250
        }
      ]
    },
    {
      "judge_id": "jdg-live-gemini-3-8-flash-20260902-or-v1",
      "model_id": "google/gemini-3.8-flash-20260902",
      "model_label": "google/gemini-3.8-flash-20260902 @ gemini-3.8-flash-20260902 via OpenRouter",
      "gateway": "openrouter",
      "people_score": {
        "state": "collecting_votes",
        "tier": null,
        "position": null,
        "rating": null,
        "ci_low": null,
        "ci_high": null,
        "decided_votes": 0,
        "keys_n": 0,
        "coarse_netfams_n": 0
      },
      "answer_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "ok_tasks": 750,
        "infra_failures": 0,
        "status_breakdown": {
          "none": 0,
          "refusal": 0,
          "invalid": 0,
          "truncated": 0,
          "error": 0,
          "provider_mismatch": 0
        },
        "estimate": 1,
        "ci_low": 1,
        "ci_high": 1
      },
      "reference_list_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "matched_brands": 1862,
        "named_brands": 2138,
        "unmatched_brands": 276,
        "estimate": 0.870907,
        "ci_low": 0.858206,
        "ci_high": 0.883486
      },
      "consistency": {
        "state": "published",
        "replicates_r": 3,
        "eligible_tasks": 750,
        "consistent_tasks": 644,
        "excluded_incomplete_tasks": 0,
        "estimate": 0.858667,
        "ci_low": 0.833333,
        "ci_high": 0.882667,
        "pairwise_agreement": 0.904,
        "pairwise_ci_low": 0.886667,
        "pairwise_ci_high": 0.92
      },
      "tailoring": {
        "state": "published",
        "eligible_pairs": 1250,
        "differing_pairs": 788,
        "clusters_n": 125,
        "estimate": 0.6304,
        "ci_low": 0.5872,
        "ci_high": 0.67122
      },
      "served_provider": "google-ai-studio",
      "served_providers": [
        {
          "provider": "google-ai-studio",
          "answers": 2250
        }
      ]
    },
    {
      "judge_id": "jdg-live-gemini-4-argon-20261013-or-v1",
      "model_id": "google/gemini-4-argon-20261013",
      "model_label": "google/gemini-4-argon-20261013 @ gemini-4-argon-20261013 via OpenRouter",
      "gateway": "openrouter",
      "people_score": {
        "state": "collecting_votes",
        "tier": null,
        "position": null,
        "rating": null,
        "ci_low": null,
        "ci_high": null,
        "decided_votes": 0,
        "keys_n": 0,
        "coarse_netfams_n": 0
      },
      "answer_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "ok_tasks": 747,
        "infra_failures": 0,
        "status_breakdown": {
          "none": 0,
          "refusal": 0,
          "invalid": 3,
          "truncated": 0,
          "error": 0,
          "provider_mismatch": 0
        },
        "estimate": 0.996,
        "ci_low": 0.990667,
        "ci_high": 1
      },
      "reference_list_rate": {
        "state": "published",
        "eligible_tasks": 747,
        "matched_brands": 1825,
        "named_brands": 2123,
        "unmatched_brands": 298,
        "estimate": 0.859633,
        "ci_low": 0.847287,
        "ci_high": 0.872522
      },
      "consistency": {
        "state": "published",
        "replicates_r": 3,
        "eligible_tasks": 745,
        "consistent_tasks": 635,
        "excluded_incomplete_tasks": 5,
        "estimate": 0.852349,
        "ci_low": 0.826846,
        "ci_high": 0.87651,
        "pairwise_agreement": 0.899329,
        "pairwise_ci_low": 0.881879,
        "pairwise_ci_high": 0.916331
      },
      "tailoring": {
        "state": "published",
        "eligible_pairs": 1242,
        "differing_pairs": 829,
        "clusters_n": 125,
        "estimate": 0.667472,
        "ci_low": 0.624398,
        "ci_high": 0.707865
      },
      "served_provider": "google-ai-studio",
      "served_providers": [
        {
          "provider": "google-ai-studio",
          "answers": 2068
        },
        {
          "provider": "google-vertex",
          "answers": 182
        }
      ]
    },
    {
      "judge_id": "jdg-live-mistral-large-4-0-20261006-or-v2",
      "model_id": "mistralai/mistral-large-4-0-20261006",
      "model_label": "mistralai/mistral-large-4-0-20261006 @ mistral-large-4-0-20261006 via OpenRouter",
      "gateway": "openrouter",
      "people_score": {
        "state": "collecting_votes",
        "tier": null,
        "position": null,
        "rating": null,
        "ci_low": null,
        "ci_high": null,
        "decided_votes": 0,
        "keys_n": 0,
        "coarse_netfams_n": 0
      },
      "answer_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "ok_tasks": 750,
        "infra_failures": 0,
        "status_breakdown": {
          "none": 0,
          "refusal": 0,
          "invalid": 0,
          "truncated": 0,
          "error": 0,
          "provider_mismatch": 0
        },
        "estimate": 1,
        "ci_low": 1,
        "ci_high": 1
      },
      "reference_list_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "matched_brands": 1932,
        "named_brands": 2150,
        "unmatched_brands": 218,
        "estimate": 0.898605,
        "ci_low": 0.887534,
        "ci_high": 0.910359
      },
      "consistency": {
        "state": "published",
        "replicates_r": 3,
        "eligible_tasks": 750,
        "consistent_tasks": 708,
        "excluded_incomplete_tasks": 0,
        "estimate": 0.944,
        "ci_low": 0.926667,
        "ci_high": 0.96,
        "pairwise_agreement": 0.962667,
        "pairwise_ci_low": 0.951111,
        "pairwise_ci_high": 0.973333
      },
      "tailoring": {
        "state": "published",
        "eligible_pairs": 1250,
        "differing_pairs": 676,
        "clusters_n": 125,
        "estimate": 0.5408,
        "ci_low": 0.4888,
        "ci_high": 0.592
      },
      "served_provider": "mistral",
      "served_providers": [
        {
          "provider": "mistral",
          "answers": 2250
        }
      ]
    },
    {
      "judge_id": "jdg-live-gpt-6-luna-20260922-or-v1",
      "model_id": "openai/gpt-6-luna-20260922",
      "model_label": "openai/gpt-6-luna-20260922 @ gpt-6-luna-20260922 via OpenRouter",
      "gateway": "openrouter",
      "people_score": {
        "state": "collecting_votes",
        "tier": null,
        "position": null,
        "rating": null,
        "ci_low": null,
        "ci_high": null,
        "decided_votes": 0,
        "keys_n": 0,
        "coarse_netfams_n": 0
      },
      "answer_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "ok_tasks": 748,
        "infra_failures": 0,
        "status_breakdown": {
          "none": 0,
          "refusal": 0,
          "invalid": 2,
          "truncated": 0,
          "error": 0,
          "provider_mismatch": 0
        },
        "estimate": 0.997333,
        "ci_low": 0.993333,
        "ci_high": 1
      },
      "reference_list_rate": {
        "state": "published",
        "eligible_tasks": 748,
        "matched_brands": 1843,
        "named_brands": 2105,
        "unmatched_brands": 262,
        "estimate": 0.875534,
        "ci_low": 0.86292,
        "ci_high": 0.888098
      },
      "consistency": {
        "state": "published",
        "replicates_r": 3,
        "eligible_tasks": 745,
        "consistent_tasks": 519,
        "excluded_incomplete_tasks": 5,
        "estimate": 0.696644,
        "ci_low": 0.663087,
        "ci_high": 0.728859,
        "pairwise_agreement": 0.788367,
        "pairwise_ci_low": 0.765101,
        "pairwise_ci_high": 0.811186
      },
      "tailoring": {
        "state": "published",
        "eligible_pairs": 1242,
        "differing_pairs": 820,
        "clusters_n": 125,
        "estimate": 0.660225,
        "ci_low": 0.610211,
        "ci_high": 0.705173
      },
      "served_provider": "openai",
      "served_providers": [
        {
          "provider": "openai",
          "answers": 2250
        }
      ]
    },
    {
      "judge_id": "jdg-live-gpt-6-1-sol-20260929-or-v1",
      "model_id": "openai/gpt-6.1-sol-20260929",
      "model_label": "openai/gpt-6.1-sol-20260929 @ gpt-6.1-sol-20260929 via OpenRouter",
      "gateway": "openrouter",
      "people_score": {
        "state": "collecting_votes",
        "tier": null,
        "position": null,
        "rating": null,
        "ci_low": null,
        "ci_high": null,
        "decided_votes": 0,
        "keys_n": 0,
        "coarse_netfams_n": 0
      },
      "answer_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "ok_tasks": 750,
        "infra_failures": 0,
        "status_breakdown": {
          "none": 0,
          "refusal": 0,
          "invalid": 0,
          "truncated": 0,
          "error": 0,
          "provider_mismatch": 0
        },
        "estimate": 1,
        "ci_low": 1,
        "ci_high": 1
      },
      "reference_list_rate": {
        "state": "published",
        "eligible_tasks": 750,
        "matched_brands": 1676,
        "named_brands": 1988,
        "unmatched_brands": 312,
        "estimate": 0.843058,
        "ci_low": 0.82838,
        "ci_high": 0.857287
      },
      "consistency": {
        "state": "published",
        "replicates_r": 3,
        "eligible_tasks": 749,
        "consistent_tasks": 591,
        "excluded_incomplete_tasks": 1,
        "estimate": 0.789052,
        "ci_low": 0.758311,
        "ci_high": 0.815754,
        "pairwise_agreement": 0.855363,
        "pairwise_ci_low": 0.834001,
        "pairwise_ci_high": 0.874054
      },
      "tailoring": {
        "state": "published",
        "eligible_pairs": 1250,
        "differing_pairs": 913,
        "clusters_n": 125,
        "estimate": 0.7304,
        "ci_low": 0.6888,
        "ci_high": 0.7712
      },
      "served_provider": "openai",
      "served_providers": [
        {
          "provider": "openai",
          "answers": 2250
        }
      ]
    }
  ]
}