{
  "_what": "The verifier's refusal rate on model designs, split by who wrote the design: a frontier model asked for architectures from a sentence, our own seeded generator, or the maintainers of 59 open-source PyTorch repositories.",
  "_notMeasured": "Whether a design that passes trains well. A refusal is the pre-flight verdict that the graph would not forward-pass, not an observed crash, except where the rule is grounded by the 264-graph study. See /calibration.json for how little the verdict says about outcome.",
  "_method": "Agent and generator rows are read from the proposer bench result files, one row per author, task set and recorded run; the generator is deterministic, so identical slates on the same sentences collapse to one row listing the dates. The human row is pinned from the MCP repository's real-repos study and checked against that checkout by test. Quoted dollars are A10G prices for one run of each refused design and are never summed across rows from different task sets.",
  "generatedFrom": [
    "scripts/rl-benchmark/proposer-results.json",
    "scripts/rl-benchmark/proposer-results.2026-09-06.json",
    "scripts/rl-benchmark/proposer-results.2026-09-04.json",
    "https://github.com/neurarch-ai/neurarch-mcp/blob/main/docs/REAL_REPOS_STUDY.md (2026-09-09)"
  ],
  "rows": [
    {
      "author": "agent",
      "who": "xai",
      "model": "grok-4",
      "modelSource": "result file",
      "taskSet": {
        "label": "stated-workload",
        "sentences": [
          "rank the next products to show a user from a 256-feature vector, over 2 billion impressions",
          "sort a support conversation into one of 60 queues from its first 512 tokens, 3M documents",
          "score card transactions for fraud from a 200-feature vector, 40M transactions",
          "find surface defects in 512x512 colour line-scan photos, 800k images"
        ]
      },
      "runAt": "2026-09-07",
      "identicalRunAt": [],
      "source": "scripts/rl-benchmark/proposer-results.json",
      "returned": 72,
      "blocked": 19,
      "blockedShare": 0.2639,
      "blockedShareByRun": [
        0.1667,
        0.3333,
        0.2917
      ],
      "blockedShareRange": [
        0.1667,
        0.3333
      ],
      "runs": 3,
      "distinct": 69,
      "novel": 69,
      "topBlockers": [
        {
          "ruleId": "structure-missing-params",
          "title": "\"embedding\" is missing numEmbeddings",
          "graphs": 13
        },
        {
          "ruleId": "invalid-output-shape",
          "title": "Shape problem: avgpool2d",
          "graphs": 3
        },
        {
          "ruleId": "unknown-layer-type",
          "title": "Shape problem: gap",
          "graphs": 3
        },
        {
          "ruleId": "merge-shape-mismatch",
          "title": "Shape problem: add",
          "graphs": 2
        }
      ],
      "quotedUsd": {
        "n": 18,
        "median": 1.9419,
        "sum": 53.4812,
        "unpriceable": 1
      },
      "handJudged": null
    },
    {
      "author": "agent",
      "who": "openrouter",
      "model": "deepseek/deepseek-chat",
      "modelSource": "result file",
      "taskSet": {
        "label": "toy",
        "sentences": [
          "classify 32x32 colour images into 10 classes",
          "recognise handwritten digits",
          "sort 16x16 RGB product thumbnails into 5 categories",
          "tell spam apart from real email from a 300-dim feature vector"
        ]
      },
      "runAt": "2026-09-06",
      "identicalRunAt": [],
      "source": "scripts/rl-benchmark/proposer-results.2026-09-06.json",
      "returned": 72,
      "blocked": 10,
      "blockedShare": 0.1389,
      "blockedShareByRun": [
        0.0833,
        0.125,
        0.2083
      ],
      "blockedShareRange": [
        0.0833,
        0.2083
      ],
      "runs": 3,
      "distinct": 64,
      "novel": 64,
      "topBlockers": [
        {
          "ruleId": "merge-shape-mismatch",
          "title": "Shape problem: add",
          "graphs": 8
        },
        {
          "ruleId": "unknown-layer-type",
          "title": "Shape problem: select",
          "graphs": 2
        }
      ],
      "quotedUsd": {
        "n": 10,
        "median": 0.0423,
        "sum": 0.4238,
        "unpriceable": 0
      },
      "handJudged": null
    },
    {
      "author": "agent",
      "who": "anthropic",
      "model": "claude-sonnet-4-6",
      "modelSource": "proposer.ts default (PROPOSER.md 2026-09-04)",
      "taskSet": {
        "label": "toy",
        "sentences": [
          "classify 32x32 colour images into 10 classes",
          "recognise handwritten digits",
          "sort 16x16 RGB product thumbnails into 5 categories",
          "tell spam apart from real email from a 300-dim feature vector"
        ]
      },
      "runAt": "2026-09-04",
      "identicalRunAt": [],
      "source": "scripts/rl-benchmark/proposer-results.2026-09-04.json",
      "returned": 66,
      "blocked": 12,
      "blockedShare": 0.1818,
      "blockedShareByRun": [
        0.0833,
        0.2917,
        0.1667
      ],
      "blockedShareRange": [
        0.0833,
        0.2917
      ],
      "runs": 3,
      "distinct": 66,
      "novel": 66,
      "topBlockers": [
        {
          "ruleId": null,
          "title": "\"trans1\" is missing outChannels, kernelSize",
          "graphs": null
        },
        {
          "ruleId": null,
          "title": "Shape problem: add",
          "graphs": null
        },
        {
          "ruleId": null,
          "title": "Shape problem: concatenate",
          "graphs": null
        },
        {
          "ruleId": null,
          "title": "Shape problem: activation",
          "graphs": null
        }
      ],
      "quotedUsd": null,
      "handJudged": null
    },
    {
      "author": "agent",
      "who": "xai",
      "model": "grok-4",
      "modelSource": "proposer.ts default (PROPOSER.md 2026-09-04)",
      "taskSet": {
        "label": "toy",
        "sentences": [
          "classify 32x32 colour images into 10 classes",
          "recognise handwritten digits",
          "sort 16x16 RGB product thumbnails into 5 categories",
          "tell spam apart from real email from a 300-dim feature vector"
        ]
      },
      "runAt": "2026-09-04",
      "identicalRunAt": [],
      "source": "scripts/rl-benchmark/proposer-results.2026-09-04.json",
      "returned": 72,
      "blocked": 2,
      "blockedShare": 0.0278,
      "blockedShareByRun": [
        0.0417,
        0.0417,
        0
      ],
      "blockedShareRange": [
        0,
        0.0417
      ],
      "runs": 3,
      "distinct": 70,
      "novel": 70,
      "topBlockers": [
        {
          "ruleId": null,
          "title": "Shape problem: maxpool2d",
          "graphs": null
        },
        {
          "ruleId": null,
          "title": "Shape problem: conv2d",
          "graphs": null
        }
      ],
      "quotedUsd": null,
      "handJudged": null
    },
    {
      "author": "generator",
      "who": "neurarch-seeded",
      "model": null,
      "modelSource": null,
      "taskSet": {
        "label": "stated-workload",
        "sentences": [
          "rank the next products to show a user from a 256-feature vector, over 2 billion impressions",
          "sort a support conversation into one of 60 queues from its first 512 tokens, 3M documents",
          "score card transactions for fraud from a 200-feature vector, 40M transactions",
          "find surface defects in 512x512 colour line-scan photos, 800k images"
        ]
      },
      "runAt": "2026-09-07",
      "identicalRunAt": [],
      "source": "scripts/rl-benchmark/proposer-results.json",
      "returned": 40,
      "blocked": 0,
      "blockedShare": 0,
      "blockedShareByRun": [
        0
      ],
      "blockedShareRange": [
        0,
        0
      ],
      "runs": 1,
      "distinct": 23,
      "novel": 0,
      "topBlockers": [],
      "quotedUsd": {
        "n": 0,
        "median": 0,
        "sum": 0,
        "unpriceable": 0
      },
      "handJudged": null
    },
    {
      "author": "generator",
      "who": "neurarch-seeded",
      "model": null,
      "modelSource": null,
      "taskSet": {
        "label": "toy",
        "sentences": [
          "classify 32x32 colour images into 10 classes",
          "recognise handwritten digits",
          "sort 16x16 RGB product thumbnails into 5 categories",
          "tell spam apart from real email from a 300-dim feature vector"
        ]
      },
      "runAt": "2026-09-06",
      "identicalRunAt": [
        "2026-09-04"
      ],
      "source": "scripts/rl-benchmark/proposer-results.2026-09-06.json",
      "returned": 56,
      "blocked": 1,
      "blockedShare": 0.0179,
      "blockedShareByRun": [
        0.0179
      ],
      "blockedShareRange": [
        0.0179,
        0.0179
      ],
      "runs": 1,
      "distinct": 29,
      "novel": 0,
      "topBlockers": [
        {
          "ruleId": "invalid-output-shape",
          "title": "Shape problem: maxpool2d",
          "graphs": 1
        }
      ],
      "quotedUsd": {
        "n": 1,
        "median": 0.0471,
        "sum": 0.0471,
        "unpriceable": 0
      },
      "handJudged": null
    },
    {
      "author": "human",
      "who": "open-source-maintainers",
      "model": null,
      "modelSource": null,
      "taskSet": {
        "label": "open-source-repositories",
        "sentences": []
      },
      "runAt": "2026-09-09",
      "identicalRunAt": [],
      "source": "https://github.com/neurarch-ai/neurarch-mcp/blob/main/docs/REAL_REPOS_STUDY.md",
      "returned": 100,
      "blocked": 3,
      "blockedShare": 0.03,
      "blockedShareByRun": [
        0.03
      ],
      "blockedShareRange": [
        0.03,
        0.03
      ],
      "runs": 1,
      "distinct": null,
      "novel": null,
      "topBlockers": [],
      "quotedUsd": null,
      "handJudged": {
        "repositories": 59,
        "filesListed": 116,
        "filesParsed": 100,
        "filesWithTenPlusLayers": 47,
        "findings": {
          "block": 3,
          "warn": 34,
          "info": 83
        },
        "blockWarn": 37,
        "artefact": 24,
        "notABug": 13,
        "realDefects": 0
      }
    }
  ]
}