{
  "generatedFrom": "scripts/rl-benchmark/SELECTORS.md, npm run bench:selectors",
  "contract": "A selector sees task, designer, staticScore and params, answers pairwise, and may abstain. Ties are scored as 0.5. Rows under 10 decided pairs are not quotable.",
  "boards": [
    {
      "label": "In-sample campaign (2026-08-21, T4)",
      "split": "in-sample",
      "designs": 24,
      "tasks": 6,
      "selectors": [
        {
          "id": "static-verifier",
          "kind": "verifier",
          "pairwise": {
            "judged": 36,
            "decided": 3,
            "correct": 2,
            "wrong": 1,
            "coverage": 0.08333333333333333,
            "accuracyWithTiesAsHalf": 0.5138888888888888,
            "accuracyWhenDecided": 0.6666666666666666,
            "quotable": false
          },
          "selectionValue": {
            "random": 0.5227897585695248,
            "picked": 0.5329450140705251,
            "oracle": 1,
            "captured": 0.021280464288777824
          },
          "costAwareSweep": [
            {
              "alpha": 1,
              "picked": 0.5329450140705264
            },
            {
              "alpha": 0.8,
              "picked": 0.31117318193296695
            },
            {
              "alpha": 0.6,
              "picked": 0.08940134979540725
            },
            {
              "alpha": 0.4,
              "picked": -0.13237048234215235
            },
            {
              "alpha": 0.2,
              "picked": -0.354142314479712
            }
          ]
        },
        {
          "id": "static-then-smallest",
          "kind": "verifier",
          "pairwise": {
            "judged": 36,
            "decided": 30,
            "correct": 14,
            "wrong": 16,
            "coverage": 0.8333333333333334,
            "accuracyWithTiesAsHalf": 0.4722222222222222,
            "accuracyWhenDecided": 0.4666666666666667,
            "quotable": true
          },
          "selectionValue": {
            "random": 0.5227897585695248,
            "picked": 0.4859584091720581,
            "oracle": 1,
            "captured": -0.0771805510440468
          },
          "costAwareSweep": [
            {
              "alpha": 1,
              "picked": 0.4859584091720552
            },
            {
              "alpha": 0.8,
              "picked": 0.3059675820384989
            },
            {
              "alpha": 0.6,
              "picked": 0.1259767549049425
            },
            {
              "alpha": 0.4,
              "picked": -0.05401407222861382
            },
            {
              "alpha": 0.2,
              "picked": -0.23400489936217017
            }
          ]
        },
        {
          "id": "smallest-params",
          "kind": "trivial",
          "pairwise": {
            "judged": 36,
            "decided": 30,
            "correct": 14,
            "wrong": 16,
            "coverage": 0.8333333333333334,
            "accuracyWithTiesAsHalf": 0.4722222222222222,
            "accuracyWhenDecided": 0.4666666666666667,
            "quotable": true
          },
          "selectionValue": {
            "random": 0.5227897585695248,
            "picked": 0.4859584091720581,
            "oracle": 1,
            "captured": -0.0771805510440468
          },
          "costAwareSweep": [
            {
              "alpha": 1,
              "picked": 0.4859584091720552
            },
            {
              "alpha": 0.8,
              "picked": 0.3059675820384989
            },
            {
              "alpha": 0.6,
              "picked": 0.1259767549049425
            },
            {
              "alpha": 0.4,
              "picked": -0.05401407222861382
            },
            {
              "alpha": 0.2,
              "picked": -0.23400489936217017
            }
          ]
        },
        {
          "id": "largest-params",
          "kind": "trivial",
          "pairwise": {
            "judged": 36,
            "decided": 30,
            "correct": 16,
            "wrong": 14,
            "coverage": 0.8333333333333334,
            "accuracyWithTiesAsHalf": 0.5277777777777778,
            "accuracyWhenDecided": 0.5333333333333333,
            "quotable": true
          },
          "selectionValue": {
            "random": 0.5227897585695248,
            "picked": 0.5110280911757737,
            "oracle": 1,
            "captured": -0.02464672040250968
          },
          "costAwareSweep": [
            {
              "alpha": 1,
              "picked": 0.5110280911757737
            },
            {
              "alpha": 0.8,
              "picked": 0.28121830627395233
            },
            {
              "alpha": 0.6,
              "picked": 0.05140852137213089
            },
            {
              "alpha": 0.4,
              "picked": -0.17840126352969046
            },
            {
              "alpha": 0.2,
              "picked": -0.40821104843151196
            }
          ]
        }
      ],
      "bounds": [
        {
          "alpha": 1,
          "random": 0.5227897585695263,
          "oracle": 1
        },
        {
          "alpha": 0.8,
          "random": 0.3000331045162938,
          "oracle": 0.7033211977686964
        },
        {
          "alpha": 0.6,
          "random": 0.0772764504630611,
          "oracle": 0.47114665777175374
        },
        {
          "alpha": 0.4,
          "random": -0.14548020359017144,
          "oracle": 0.29389575164581566
        },
        {
          "alpha": 0.2,
          "random": -0.36823685764340414,
          "oracle": 0.13491162546890048
        }
      ]
    },
    {
      "label": "Out-of-sample campaign (2026-09-01, T4)",
      "split": "out-of-sample",
      "designs": 15,
      "tasks": 5,
      "selectors": [
        {
          "id": "static-verifier",
          "kind": "verifier",
          "pairwise": {
            "judged": 15,
            "decided": 4,
            "correct": 3,
            "wrong": 1,
            "coverage": 0.26666666666666666,
            "accuracyWithTiesAsHalf": 0.5666666666666667,
            "accuracyWhenDecided": 0.75,
            "quotable": false
          },
          "selectionValue": {
            "random": 0.5130232980690709,
            "picked": 0.6192021394792292,
            "oracle": 1,
            "captured": 0.21803679927426653
          },
          "costAwareSweep": [
            {
              "alpha": 1,
              "picked": 0.6192021394792309
            },
            {
              "alpha": 0.8,
              "picked": 0.4525045687262419
            },
            {
              "alpha": 0.6,
              "picked": 0.2858069979732528
            },
            {
              "alpha": 0.4,
              "picked": 0.11910942722026382
            },
            {
              "alpha": 0.2,
              "picked": -0.047588143532725245
            }
          ]
        },
        {
          "id": "static-then-smallest",
          "kind": "verifier",
          "pairwise": {
            "judged": 15,
            "decided": 15,
            "correct": 5,
            "wrong": 10,
            "coverage": 1,
            "accuracyWithTiesAsHalf": 0.3333333333333333,
            "accuracyWhenDecided": 0.3333333333333333,
            "quotable": true
          },
          "selectionValue": {
            "random": 0.5130232980690709,
            "picked": 0.515970515970516,
            "oracle": 1,
            "captured": 0.006052071669463901
          },
          "costAwareSweep": [
            {
              "alpha": 1,
              "picked": 0.515970515970516
            },
            {
              "alpha": 0.8,
              "picked": 0.3838875238875239
            },
            {
              "alpha": 0.6,
              "picked": 0.2518045318045318
            },
            {
              "alpha": 0.4,
              "picked": 0.11972153972153973
            },
            {
              "alpha": 0.2,
              "picked": -0.012361452361452363
            }
          ]
        },
        {
          "id": "smallest-params",
          "kind": "trivial",
          "pairwise": {
            "judged": 15,
            "decided": 15,
            "correct": 4,
            "wrong": 11,
            "coverage": 1,
            "accuracyWithTiesAsHalf": 0.26666666666666666,
            "accuracyWhenDecided": 0.26666666666666666,
            "quotable": true
          },
          "selectionValue": {
            "random": 0.5130232980690709,
            "picked": 0.315970515970516,
            "oracle": 1,
            "captured": -0.4046451941483313
          },
          "costAwareSweep": [
            {
              "alpha": 1,
              "picked": 0.315970515970516
            },
            {
              "alpha": 0.8,
              "picked": 0.2238875238875239
            },
            {
              "alpha": 0.6,
              "picked": 0.1318045318045318
            },
            {
              "alpha": 0.4,
              "picked": 0.03972153972153974
            },
            {
              "alpha": 0.2,
              "picked": -0.052361452361452364
            }
          ]
        },
        {
          "id": "largest-params",
          "kind": "trivial",
          "pairwise": {
            "judged": 15,
            "decided": 15,
            "correct": 11,
            "wrong": 4,
            "coverage": 1,
            "accuracyWithTiesAsHalf": 0.7333333333333333,
            "accuracyWhenDecided": 0.7333333333333333,
            "quotable": true
          },
          "selectionValue": {
            "random": 0.5130232980690709,
            "picked": 0.8389859030593971,
            "oracle": 1,
            "captured": 0.6693597531418649
          },
          "costAwareSweep": [
            {
              "alpha": 1,
              "picked": 0.8389859030593971
            },
            {
              "alpha": 0.8,
              "picked": 0.6311887224475179
            },
            {
              "alpha": 0.6,
              "picked": 0.4233915418356383
            },
            {
              "alpha": 0.4,
              "picked": 0.2155943612237589
            },
            {
              "alpha": 0.2,
              "picked": 0.007797180611879434
            }
          ]
        },
        {
          "id": "judge:claude-opus-5",
          "kind": "judge",
          "pairwise": {
            "judged": 15,
            "decided": 15,
            "correct": 12,
            "wrong": 3,
            "coverage": 1,
            "accuracyWithTiesAsHalf": 0.8,
            "accuracyWhenDecided": 0.8,
            "quotable": true
          },
          "selectionValue": {
            "random": 0.5130232980690709,
            "picked": 0.9255813953488398,
            "oracle": 1,
            "captured": 0.8471824127189652
          },
          "costAwareSweep": [
            {
              "alpha": 1,
              "picked": 0.9255813953488398
            },
            {
              "alpha": 0.8,
              "picked": 0.7004651162790718
            },
            {
              "alpha": 0.6,
              "picked": 0.4753488372093038
            },
            {
              "alpha": 0.4,
              "picked": 0.25023255813953593
            },
            {
              "alpha": 0.2,
              "picked": 0.02511627906976794
            }
          ]
        },
        {
          "id": "judge:grok-4.3",
          "kind": "judge",
          "pairwise": {
            "judged": 15,
            "decided": 15,
            "correct": 11,
            "wrong": 4,
            "coverage": 1,
            "accuracyWithTiesAsHalf": 0.7333333333333333,
            "accuracyWhenDecided": 0.7333333333333333,
            "quotable": true
          },
          "selectionValue": {
            "random": 0.5130232980690709,
            "picked": 0.8455813953488397,
            "oracle": 1,
            "captured": 0.6829035063918469
          },
          "costAwareSweep": [
            {
              "alpha": 1,
              "picked": 0.8455813953488398
            },
            {
              "alpha": 0.8,
              "picked": 0.6364651162790718
            },
            {
              "alpha": 0.6,
              "picked": 0.42734883720930383
            },
            {
              "alpha": 0.4,
              "picked": 0.2182325581395359
            },
            {
              "alpha": 0.2,
              "picked": 0.009116279069767943
            }
          ]
        }
      ],
      "bounds": [
        {
          "alpha": 1,
          "random": 0.5130232980690728,
          "oracle": 1
        },
        {
          "alpha": 0.8,
          "random": 0.3734874215240413,
          "oracle": 0.76
        },
        {
          "alpha": 0.6,
          "random": 0.23395154497900977,
          "oracle": 0.52
        },
        {
          "alpha": 0.4,
          "random": 0.09441566843397833,
          "oracle": 0.3239311043566363
        },
        {
          "alpha": 0.2,
          "random": -0.045120208111053176,
          "oracle": 0.16
        }
      ]
    }
  ]
}