{
  "verification": {
    "reference_model": "jev-1.13.0",
    "public_revision": "bb05a335bc809e61b20c0f745d25499a82b326fc",
    "public_artifact_revision": "v1.3.0",
    "public_source": "https://github.com/fstandhartinger/jevbench/blob/bb05a335bc809e61b20c0f745d25499a82b326fc/results/v1.2/jevbench-v1.2-per-task.json",
    "public_case_count": 231,
    "public_jev_correct": 200,
    "public_jev_correct_by_tier": {
      "easy": 48,
      "original": 71,
      "hard": 81
    },
    "public_cases_match_existing_fixture": true,
    "pilot_revision": "0d610cc53e79bcbec691312b0c4adb4a0e371642",
    "pilot_source": "https://github.com/AbdelStark/jev-benchmarks/blob/0d610cc53e79bcbec691312b0c4adb4a0e371642/results/reports/btzsc-pilot-v1.md",
    "pilot_case_count": 300,
    "pilot_manifest_matches_published_hash": true,
    "pilot_published_accuracy": {
      "agnews": 0.91,
      "banking77": 0.87,
      "emotiondair": 0.48
    },
    "verified_source_sha256": {
      "results/v1.2/jevbench-v1.2-per-task.json": "c772b3b85809f483449ec2a6ae0272347371ed5f40f7faff8394ecf4ee22ae7c",
      "datasets/manifest.json": "0983ad3e73a34e7c55b02b5d09ed983aeccabef0253bb31be43a2e6230d58f7b",
      "datasets/public/easy.jsonl": "231df3c2c8e88a1a8c137ebe85de96ba70fabd330849098ac7b3c52c70b7172b",
      "datasets/public/original.jsonl": "5c2414edb3006b8bfcb70fda433f0f9ca015759433849f8d3104328a1f7c4180",
      "datasets/public/hard.jsonl": "89e9e6becb33ed88c1de7d42dcc87531b2fb64cfaef4e1986faf7c37b3f80ebb",
      "pilot/configs/pilot-v1.yaml": "bb9172b7b8d92331ab166f13642ecc8aac4d447f111a0daf3285ab31439c05ef",
      "pilot/results/reports/btzsc-pilot-v1.json": "292db65ade535b9cd9c06477b6f7863e03a6106ec0a5fa56378a08164ee14598",
      "pilot/src/jev_benchmarks/adapters/jev.py": "5055ff17fe88120a18cbeda9ecfa5f7bf3111c8d53cb20fecc8b8657099e083b",
      "pilot/src/jev_benchmarks/metrics.py": "bd36d82ad3525ad2cc16284cf720d3d29aa0669f1d7a48a8dc5846217d04173b",
      "pilot/src/jev_benchmarks/runner.py": "e6a3075cfc23818de59f121e21799d5e3ee25b00be95f0c7d1d28a95f85627e6"
    },
    "limitations": [
      "Reference outcomes are independently published observations, not a new Jev API run.",
      "Public and sealed JevBench scores must not be conflated.",
      "Public outcomes permit paired correctness comparisons, but not paired calibration without Jev probability vectors.",
      "Pilot per-case predictions are not published; aggregate differences only."
    ]
  },
  "models": {
    "clm-int8": {
      "label": "CLM 8B INT8",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 33,
          "jev_correct": 48,
          "local_accuracy": 0.6875,
          "jev_accuracy": 1.0,
          "difference_pp": -31.25,
          "paired_cluster_bootstrap95_pp": [
            -43.75,
            -18.75
          ],
          "local_only_correct": 0,
          "jev_only_correct": 15,
          "both_correct": 33,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 25,
          "jev_correct": 71,
          "local_accuracy": 0.3472222222222222,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -63.888888888888886,
          "paired_cluster_bootstrap95_pp": [
            -77.77777777777777,
            -50.0
          ],
          "local_only_correct": 1,
          "jev_only_correct": 47,
          "both_correct": 24,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 38,
          "jev_correct": 81,
          "local_accuracy": 0.34234234234234234,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -38.73873873873874,
          "paired_cluster_bootstrap95_pp": [
            -51.351351351351354,
            -25.225225225225227
          ],
          "local_only_correct": 12,
          "jev_only_correct": 55,
          "both_correct": 26,
          "both_wrong": 18,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 96,
          "jev_correct": 200,
          "local_accuracy": 0.4155844155844156,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -45.02164502164502,
          "paired_cluster_bootstrap95_pp": [
            -53.246753246753244,
            -37.22943722943723
          ],
          "local_only_correct": 13,
          "jev_only_correct": 117,
          "both_correct": 83,
          "both_wrong": 18,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.47,
          "macro_f1": 0.42125795479919614,
          "brier": 0.6306514566896155,
          "nll": 1.1319258301385338,
          "ece": 0.18336878481634097,
          "mean_confidence": 0.6268140824258365,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.23,
          "risk_at_selected_coverage": 0.043478260869565216,
          "confidence_threshold_at_error_budget": 0.799307061260239,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -44.00000000000001
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.22,
          "macro_f1": 0.12657798643714135,
          "brier": 0.9806249998594363,
          "nll": 2.212353602670595,
          "ece": 0.34438478548598284,
          "mean_confidence": 0.5616554341631192,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.01,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.8635324338568263,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -26.0
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.08,
          "macro_f1": 0.04681187150699346,
          "brier": 0.9869446384183361,
          "nll": 4.249826097414273,
          "ece": 0.11632673403685337,
          "mean_confidence": 0.17753389023331903,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.03,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.37383593441650986,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -79.0
        }
      },
      "pilot_attempted": 300
    },
    "cygnet-12b-nf4": {
      "label": "Cygnet 12B NF4 (local variant)",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 65,
          "jev_correct": 71,
          "local_accuracy": 0.9027777777777778,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -8.333333333333332,
          "paired_cluster_bootstrap95_pp": [
            -16.666666666666668,
            -1.3888888888888888
          ],
          "local_only_correct": 0,
          "jev_only_correct": 6,
          "both_correct": 65,
          "both_wrong": 1,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 82,
          "jev_correct": 81,
          "local_accuracy": 0.7387387387387387,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": 0.9009009009009009,
          "paired_cluster_bootstrap95_pp": [
            -8.108108108108109,
            9.90990990990991
          ],
          "local_only_correct": 13,
          "jev_only_correct": 12,
          "both_correct": 69,
          "both_wrong": 17,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 195,
          "jev_correct": 200,
          "local_accuracy": 0.8441558441558441,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -2.1645021645021645,
          "paired_cluster_bootstrap95_pp": [
            -6.937229437229437,
            2.1645021645021645
          ],
          "local_only_correct": 13,
          "jev_only_correct": 18,
          "both_correct": 182,
          "both_wrong": 18,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.88,
          "macro_f1": 0.8729648341786616,
          "brier": 0.18699503836842463,
          "nll": 0.3635525557402627,
          "ece": 0.06577173369057589,
          "mean_confidence": 0.9227754381905658,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.76,
          "risk_at_selected_coverage": 0.039473684210526314,
          "confidence_threshold_at_error_budget": 0.9258797772469187,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -3.0000000000000027
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.43,
          "macro_f1": 0.4127180992961257,
          "brier": 0.863210483462641,
          "nll": 1.971815627578444,
          "ece": 0.36781298852344024,
          "mean_confidence": 0.7750729626579532,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.06,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9758817972220565,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -4.999999999999999
        },
        "banking77": {
          "n": 100,
          "valid": 0,
          "failures": 100,
          "accuracy": 0.0,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -87.0
        }
      },
      "pilot_attempted": 300
    },
    "decider-08b": {
      "label": "Decider 0.8B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 61,
          "jev_correct": 71,
          "local_accuracy": 0.8472222222222222,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -13.88888888888889,
          "paired_cluster_bootstrap95_pp": [
            -25.0,
            -5.555555555555555
          ],
          "local_only_correct": 1,
          "jev_only_correct": 11,
          "both_correct": 60,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 43,
          "jev_correct": 81,
          "local_accuracy": 0.38738738738738737,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -34.234234234234236,
          "paired_cluster_bootstrap95_pp": [
            -44.14414414414414,
            -23.423423423423422
          ],
          "local_only_correct": 6,
          "jev_only_correct": 44,
          "both_correct": 37,
          "both_wrong": 24,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 152,
          "jev_correct": 200,
          "local_accuracy": 0.658008658008658,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -20.77922077922078,
          "paired_cluster_bootstrap95_pp": [
            -26.83982683982684,
            -14.718614718614718
          ],
          "local_only_correct": 7,
          "jev_only_correct": 55,
          "both_correct": 145,
          "both_wrong": 24,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.81,
          "macro_f1": 0.7700289816755392,
          "brier": 0.2870154796815711,
          "nll": 0.5511622504417194,
          "ece": 0.09709680813276812,
          "mean_confidence": 0.792606096900841,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 34,
          "coverage_at_error_budget": 0.61,
          "risk_at_selected_coverage": 0.04918032786885246,
          "confidence_threshold_at_error_budget": 0.7962,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -9.999999999999998
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.61,
          "macro_f1": 0.5991302240644346,
          "brier": 0.4906732033363525,
          "nll": 0.9680137100973712,
          "ece": 0.06567568269077902,
          "mean_confidence": 0.626959022297668,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 42,
          "coverage_at_error_budget": 0.07,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9400940094009401,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 13.0
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.78,
          "macro_f1": 0.757473544973545,
          "brier": 0.3057995538573101,
          "nll": 0.8263454672371779,
          "ece": 0.05800209140943353,
          "mean_confidence": 0.8013702803828449,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 93,
          "coverage_at_error_budget": 0.65,
          "risk_at_selected_coverage": 0.046153846153846156,
          "confidence_threshold_at_error_budget": 0.7291916766706684,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -8.999999999999996
        }
      },
      "pilot_attempted": 300
    },
    "decider-2b": {
      "label": "Decider 2B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 64,
          "jev_correct": 71,
          "local_accuracy": 0.8888888888888888,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -9.722222222222223,
          "paired_cluster_bootstrap95_pp": [
            -19.444444444444443,
            -1.3888888888888888
          ],
          "local_only_correct": 1,
          "jev_only_correct": 8,
          "both_correct": 63,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 63,
          "jev_correct": 81,
          "local_accuracy": 0.5675675675675675,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -16.216216216216218,
          "paired_cluster_bootstrap95_pp": [
            -26.126126126126128,
            -6.306306306306307
          ],
          "local_only_correct": 9,
          "jev_only_correct": 27,
          "both_correct": 54,
          "both_wrong": 21,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 175,
          "jev_correct": 200,
          "local_accuracy": 0.7575757575757576,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -10.822510822510822,
          "paired_cluster_bootstrap95_pp": [
            -16.450216450216452,
            -5.194805194805195
          ],
          "local_only_correct": 10,
          "jev_only_correct": 35,
          "both_correct": 165,
          "both_wrong": 21,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.93,
          "macro_f1": 0.9273629964806436,
          "brier": 0.11677029770989243,
          "nll": 0.2583018179228778,
          "ece": 0.0662594706914347,
          "mean_confidence": 0.8738297093882972,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 36,
          "coverage_at_error_budget": 0.92,
          "risk_at_selected_coverage": 0.043478260869565216,
          "confidence_threshold_at_error_budget": 0.6400640064006401,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": 2.0000000000000018
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.76,
          "macro_f1": 0.7576637243303909,
          "brier": 0.37352661513949004,
          "nll": 0.729253789085368,
          "ece": 0.0750007590685273,
          "mean_confidence": 0.7490969088401888,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 39,
          "coverage_at_error_budget": 0.13,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9583,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 28.000000000000004
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.78,
          "macro_f1": 0.7352513227513228,
          "brier": 0.27321976952632576,
          "nll": 0.5978006278578585,
          "ece": 0.047288752410928864,
          "mean_confidence": 0.7916441602274596,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 84,
          "coverage_at_error_budget": 0.65,
          "risk_at_selected_coverage": 0.046153846153846156,
          "confidence_threshold_at_error_budget": 0.7927036481759121,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -8.999999999999996
        }
      },
      "pilot_attempted": 300
    },
    "decider-4b": {
      "label": "Decider 4B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 71,
          "jev_correct": 71,
          "local_accuracy": 0.9861111111111112,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            -4.166666666666667,
            4.166666666666667
          ],
          "local_only_correct": 1,
          "jev_only_correct": 1,
          "both_correct": 70,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 72,
          "jev_correct": 81,
          "local_accuracy": 0.6486486486486487,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -8.108108108108109,
          "paired_cluster_bootstrap95_pp": [
            -17.117117117117118,
            0.0
          ],
          "local_only_correct": 7,
          "jev_only_correct": 16,
          "both_correct": 65,
          "both_wrong": 23,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 191,
          "jev_correct": 200,
          "local_accuracy": 0.8268398268398268,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -3.896103896103896,
          "paired_cluster_bootstrap95_pp": [
            -8.225108225108226,
            0.4329004329004329
          ],
          "local_only_correct": 8,
          "jev_only_correct": 17,
          "both_correct": 183,
          "both_wrong": 23,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.96,
          "macro_f1": 0.9593627450980392,
          "brier": 0.0736295716230461,
          "nll": 0.17312745657128795,
          "ece": 0.027525224511812177,
          "mean_confidence": 0.9324747754881878,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 35,
          "coverage_at_error_budget": 1.0,
          "risk_at_selected_coverage": 0.04,
          "confidence_threshold_at_error_budget": 0.5843,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": 4.999999999999993
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.71,
          "macro_f1": 0.7075294296346928,
          "brier": 0.388493113260396,
          "nll": 0.7644768242365346,
          "ece": 0.08683264225443574,
          "mean_confidence": 0.7241675687973519,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 48,
          "coverage_at_error_budget": 0.23,
          "risk_at_selected_coverage": 0.043478260869565216,
          "confidence_threshold_at_error_budget": 0.9268000000000001,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 23.0
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.89,
          "macro_f1": 0.8675925925925926,
          "brier": 0.17110831972102858,
          "nll": 0.4208757998397742,
          "ece": 0.08608478590209326,
          "mean_confidence": 0.8432509964982563,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 93,
          "coverage_at_error_budget": 0.92,
          "risk_at_selected_coverage": 0.043478260869565216,
          "confidence_threshold_at_error_budget": 0.4371562843715631,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": 2.0000000000000018
        }
      },
      "pilot_attempted": 300
    },
    "decision-4b-v12": {
      "label": "Decision 4B v1.2",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 69,
          "jev_correct": 71,
          "local_accuracy": 0.9583333333333334,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -2.7777777777777777,
          "paired_cluster_bootstrap95_pp": [
            -9.722222222222221,
            2.7777777777777777
          ],
          "local_only_correct": 1,
          "jev_only_correct": 3,
          "both_correct": 68,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 86,
          "jev_correct": 81,
          "local_accuracy": 0.7747747747747747,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": 4.504504504504505,
          "paired_cluster_bootstrap95_pp": [
            -3.6036036036036037,
            12.612612612612613
          ],
          "local_only_correct": 12,
          "jev_only_correct": 7,
          "both_correct": 74,
          "both_wrong": 18,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 203,
          "jev_correct": 200,
          "local_accuracy": 0.8787878787878788,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": 1.2987012987012987,
          "paired_cluster_bootstrap95_pp": [
            -3.0303030303030303,
            5.627705627705628
          ],
          "local_only_correct": 13,
          "jev_only_correct": 10,
          "both_correct": 190,
          "both_wrong": 18,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.91,
          "macro_f1": 0.9038188915340997,
          "brier": 0.13965696228003768,
          "nll": 0.27633743540101774,
          "ece": 0.07681853338398026,
          "mean_confidence": 0.8460438940166344,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 98,
          "coverage_at_error_budget": 0.87,
          "risk_at_selected_coverage": 0.04597701149425287,
          "confidence_threshold_at_error_budget": 0.616756766968079,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": 0.0
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.51,
          "macro_f1": 0.5143459316093587,
          "brier": 0.6495110817302268,
          "nll": 1.331294047617886,
          "ece": 0.13065943336464456,
          "mean_confidence": 0.5550899666216188,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 97,
          "coverage_at_error_budget": 0.06,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9409729531520116,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 3.0000000000000027
        },
        "banking77": {
          "n": 100,
          "valid": 0,
          "failures": 100,
          "accuracy": 0.0,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -87.0
        }
      },
      "pilot_attempted": 300
    },
    "decision2-eos-08b": {
      "label": "Decision 2.0 Eos 0.8B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 64,
          "jev_correct": 71,
          "local_accuracy": 0.8888888888888888,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -9.722222222222223,
          "paired_cluster_bootstrap95_pp": [
            -18.055555555555557,
            -2.743055555555745
          ],
          "local_only_correct": 1,
          "jev_only_correct": 8,
          "both_correct": 63,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 51,
          "jev_correct": 81,
          "local_accuracy": 0.4594594594594595,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -27.027027027027028,
          "paired_cluster_bootstrap95_pp": [
            -37.83783783783784,
            -16.216216216216218
          ],
          "local_only_correct": 6,
          "jev_only_correct": 36,
          "both_correct": 45,
          "both_wrong": 24,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 163,
          "jev_correct": 200,
          "local_accuracy": 0.7056277056277056,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -16.017316017316016,
          "paired_cluster_bootstrap95_pp": [
            -21.645021645021647,
            -10.38961038961039
          ],
          "local_only_correct": 7,
          "jev_only_correct": 44,
          "both_correct": 156,
          "both_wrong": 24,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.73,
          "macro_f1": 0.680044275774826,
          "brier": 0.39259620589630373,
          "nll": 0.7751591026589614,
          "ece": 0.1045517548556443,
          "mean_confidence": 0.8323699882303324,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.28,
          "risk_at_selected_coverage": 0.03571428571428571,
          "confidence_threshold_at_error_budget": 0.9758365661937759,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -18.000000000000004
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.49,
          "macro_f1": 0.48719993723224436,
          "brier": 0.685764887372639,
          "nll": 1.5610186627514644,
          "ece": 0.2596564262057103,
          "mean_confidence": 0.7413192118954377,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.07,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.998974349551087,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 1.0000000000000009
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.9,
          "macro_f1": 0.8944444444444444,
          "brier": 0.16211291910225345,
          "nll": 0.4085163485527694,
          "ece": 0.07063462327294896,
          "mean_confidence": 0.8543611167892817,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.86,
          "risk_at_selected_coverage": 0.046511627906976744,
          "confidence_threshold_at_error_budget": 0.6230486332950946,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": 3.0000000000000027
        }
      },
      "pilot_attempted": 300
    },
    "decision2-kai-06b": {
      "label": "Decision 2.0 Kai 0.6B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 47,
          "jev_correct": 48,
          "local_accuracy": 0.9791666666666666,
          "jev_accuracy": 1.0,
          "difference_pp": -2.083333333333333,
          "paired_cluster_bootstrap95_pp": [
            -6.25,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 1,
          "both_correct": 47,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 61,
          "jev_correct": 71,
          "local_accuracy": 0.8472222222222222,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -13.88888888888889,
          "paired_cluster_bootstrap95_pp": [
            -23.61111111111111,
            -5.555555555555555
          ],
          "local_only_correct": 1,
          "jev_only_correct": 11,
          "both_correct": 60,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 44,
          "jev_correct": 81,
          "local_accuracy": 0.3963963963963964,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -33.33333333333333,
          "paired_cluster_bootstrap95_pp": [
            -43.24324324324324,
            -23.40090090090102
          ],
          "local_only_correct": 4,
          "jev_only_correct": 41,
          "both_correct": 40,
          "both_wrong": 26,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 152,
          "jev_correct": 200,
          "local_accuracy": 0.658008658008658,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -20.77922077922078,
          "paired_cluster_bootstrap95_pp": [
            -26.406926406926406,
            -15.151515151515152
          ],
          "local_only_correct": 5,
          "jev_only_correct": 53,
          "both_correct": 147,
          "both_wrong": 26,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.83,
          "macro_f1": 0.8157979764410472,
          "brier": 0.25782996402600844,
          "nll": 0.4620415834186572,
          "ece": 0.058847022430903125,
          "mean_confidence": 0.809309900608095,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.46,
          "risk_at_selected_coverage": 0.043478260869565216,
          "confidence_threshold_at_error_budget": 0.8775795239232824,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -8.000000000000007
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.53,
          "macro_f1": 0.5344801676533546,
          "brier": 0.606772688489892,
          "nll": 1.2287275449250075,
          "ece": 0.10603260356947054,
          "mean_confidence": 0.6277331087382787,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.11,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9108788270337579,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 5.000000000000004
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.81,
          "macro_f1": 0.8109126984126984,
          "brier": 0.25735485593227875,
          "nll": 0.6832770423050047,
          "ece": 0.13960721702872478,
          "mean_confidence": 0.7020760332578853,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.78,
          "risk_at_selected_coverage": 0.038461538461538464,
          "confidence_threshold_at_error_budget": 0.5118700254762006,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -5.999999999999995
        }
      },
      "pilot_attempted": 300
    },
    "decision2-nox-4b": {
      "label": "Decision 2.0 Nox 4B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 68,
          "jev_correct": 71,
          "local_accuracy": 0.9444444444444444,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -4.166666666666666,
          "paired_cluster_bootstrap95_pp": [
            -13.88888888888889,
            2.7777777777777777
          ],
          "local_only_correct": 1,
          "jev_only_correct": 4,
          "both_correct": 67,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 69,
          "jev_correct": 81,
          "local_accuracy": 0.6216216216216216,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -10.81081081081081,
          "paired_cluster_bootstrap95_pp": [
            -18.91891891891892,
            -3.6036036036036037
          ],
          "local_only_correct": 3,
          "jev_only_correct": 15,
          "both_correct": 66,
          "both_wrong": 27,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 185,
          "jev_correct": 200,
          "local_accuracy": 0.8008658008658008,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -6.493506493506493,
          "paired_cluster_bootstrap95_pp": [
            -10.822510822510823,
            -2.586580086580146
          ],
          "local_only_correct": 4,
          "jev_only_correct": 19,
          "both_correct": 181,
          "both_wrong": 27,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.85,
          "macro_f1": 0.829952700683251,
          "brier": 0.2137503747549807,
          "nll": 0.3627664137898304,
          "ece": 0.08115424039441353,
          "mean_confidence": 0.898053645290807,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.76,
          "risk_at_selected_coverage": 0.039473684210526314,
          "confidence_threshold_at_error_budget": 0.8425297360567833,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -6.000000000000005
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.51,
          "macro_f1": 0.512612734192676,
          "brier": 0.6377739569814431,
          "nll": 1.3464532877259883,
          "ece": 0.1319501344318774,
          "mean_confidence": 0.609915649916348,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.03,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9732928004648107,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 3.0000000000000027
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.86,
          "macro_f1": 0.8574074074074074,
          "brier": 0.2311392206802732,
          "nll": 0.5021653464675238,
          "ece": 0.11018888794998125,
          "mean_confidence": 0.7985796681363343,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.71,
          "risk_at_selected_coverage": 0.04225352112676056,
          "confidence_threshold_at_error_budget": 0.7343046174353639,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -1.0000000000000009
        }
      },
      "pilot_attempted": 300
    },
    "decision2-sol-2b": {
      "label": "Decision 2.0 Sol 2B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 67,
          "jev_correct": 71,
          "local_accuracy": 0.9305555555555556,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -5.555555555555555,
          "paired_cluster_bootstrap95_pp": [
            -13.88888888888889,
            1.3888888888888888
          ],
          "local_only_correct": 1,
          "jev_only_correct": 5,
          "both_correct": 66,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 51,
          "jev_correct": 81,
          "local_accuracy": 0.4594594594594595,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -27.027027027027028,
          "paired_cluster_bootstrap95_pp": [
            -36.93693693693694,
            -17.117117117117118
          ],
          "local_only_correct": 4,
          "jev_only_correct": 34,
          "both_correct": 47,
          "both_wrong": 26,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 166,
          "jev_correct": 200,
          "local_accuracy": 0.7186147186147186,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -14.71861471861472,
          "paired_cluster_bootstrap95_pp": [
            -19.913419913419915,
            -9.523809523809524
          ],
          "local_only_correct": 5,
          "jev_only_correct": 39,
          "both_correct": 161,
          "both_wrong": 26,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.8,
          "macro_f1": 0.7683881335553089,
          "brier": 0.3094013777239084,
          "nll": 0.5573990066870254,
          "ece": 0.07808215492913925,
          "mean_confidence": 0.8319408816778239,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.47,
          "risk_at_selected_coverage": 0.0425531914893617,
          "confidence_threshold_at_error_budget": 0.8942196545785317,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -10.999999999999998
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.52,
          "macro_f1": 0.4995189534035706,
          "brier": 0.6223056488384298,
          "nll": 1.3518112630730417,
          "ece": 0.15807219682167145,
          "mean_confidence": 0.6780721968216716,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.02,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9913146400620795,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 4.0000000000000036
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.89,
          "macro_f1": 0.8837962962962963,
          "brier": 0.19393854356598172,
          "nll": 0.4859641467382916,
          "ece": 0.12197154027504248,
          "mean_confidence": 0.7735775976295952,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.84,
          "risk_at_selected_coverage": 0.047619047619047616,
          "confidence_threshold_at_error_budget": 0.5501361748354914,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": 2.0000000000000018
        }
      },
      "pilot_attempted": 300
    },
    "gliner-decide-1b": {
      "label": "GLiNER Decide 1B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 47,
          "jev_correct": 48,
          "local_accuracy": 0.9791666666666666,
          "jev_accuracy": 1.0,
          "difference_pp": -2.083333333333333,
          "paired_cluster_bootstrap95_pp": [
            -6.25,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 1,
          "both_correct": 47,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 35,
          "jev_correct": 71,
          "local_accuracy": 0.4861111111111111,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -50.0,
          "paired_cluster_bootstrap95_pp": [
            -63.888888888888886,
            -34.72222222222222
          ],
          "local_only_correct": 0,
          "jev_only_correct": 36,
          "both_correct": 35,
          "both_wrong": 1,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 39,
          "jev_correct": 81,
          "local_accuracy": 0.35135135135135137,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -37.83783783783784,
          "paired_cluster_bootstrap95_pp": [
            -47.747747747747745,
            -27.92792792792793
          ],
          "local_only_correct": 2,
          "jev_only_correct": 44,
          "both_correct": 37,
          "both_wrong": 28,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 121,
          "jev_correct": 200,
          "local_accuracy": 0.5238095238095238,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -34.1991341991342,
          "paired_cluster_bootstrap95_pp": [
            -40.692640692640694,
            -27.705627705627705
          ],
          "local_only_correct": 2,
          "jev_only_correct": 81,
          "both_correct": 119,
          "both_wrong": 29,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.74,
          "macro_f1": 0.7239048880061789,
          "brier": 0.3297293275152435,
          "nll": 0.6002480325390072,
          "ece": 0.03859690161553491,
          "mean_confidence": 0.742912724874057,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 96,
          "coverage_at_error_budget": 0.29,
          "risk_at_selected_coverage": 0.034482758620689655,
          "confidence_threshold_at_error_budget": 0.8941293725200858,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -17.000000000000004
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.48,
          "macro_f1": 0.4544298880138479,
          "brier": 0.6543337783965968,
          "nll": 1.3798323295846442,
          "ece": 0.09074085675968668,
          "mean_confidence": 0.3892591432403134,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 89,
          "coverage_at_error_budget": 0.15,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.5601237457999866,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 0.0
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.44,
          "macro_f1": 0.38233024691358025,
          "brier": 0.8178555177566852,
          "nll": 2.8405394580185925,
          "ece": 0.2951466214689427,
          "mean_confidence": 0.14485337853105731,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 98,
          "coverage_at_error_budget": 0.18,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.22987077584331408,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -43.0
        }
      },
      "pilot_attempted": 300
    },
    "gliner-decide-multi": {
      "label": "GLiNER Decide multilingual",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 47,
          "jev_correct": 48,
          "local_accuracy": 0.9791666666666666,
          "jev_accuracy": 1.0,
          "difference_pp": -2.083333333333333,
          "paired_cluster_bootstrap95_pp": [
            -6.25,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 1,
          "both_correct": 47,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 42,
          "jev_correct": 71,
          "local_accuracy": 0.5833333333333334,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -40.27777777777778,
          "paired_cluster_bootstrap95_pp": [
            -56.94444444444444,
            -25.0
          ],
          "local_only_correct": 1,
          "jev_only_correct": 30,
          "both_correct": 41,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 42,
          "jev_correct": 81,
          "local_accuracy": 0.3783783783783784,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -35.13513513513514,
          "paired_cluster_bootstrap95_pp": [
            -45.945945945945944,
            -23.423423423423422
          ],
          "local_only_correct": 9,
          "jev_only_correct": 48,
          "both_correct": 33,
          "both_wrong": 21,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 131,
          "jev_correct": 200,
          "local_accuracy": 0.5670995670995671,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -29.87012987012987,
          "paired_cluster_bootstrap95_pp": [
            -37.22943722943723,
            -22.07792207792208
          ],
          "local_only_correct": 10,
          "jev_only_correct": 79,
          "both_correct": 121,
          "both_wrong": 21,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.78,
          "macro_f1": 0.7688676347732075,
          "brier": 0.36343620861382364,
          "nll": 0.7133837996613832,
          "ece": 0.19856706563080662,
          "mean_confidence": 0.587350340539523,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 95,
          "coverage_at_error_budget": 0.28,
          "risk_at_selected_coverage": 0.03571428571428571,
          "confidence_threshold_at_error_budget": 0.704970154726127,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -13.0
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.49,
          "macro_f1": 0.4580952078263291,
          "brier": 0.6802847049145266,
          "nll": 1.4151729153884327,
          "ece": 0.14931041926250227,
          "mean_confidence": 0.38410729080486605,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 95,
          "coverage_at_error_budget": 0.0,
          "risk_at_selected_coverage": null,
          "confidence_threshold_at_error_budget": null,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 1.0000000000000009
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.61,
          "macro_f1": 0.5880291005291005,
          "brier": 0.7111796418442353,
          "nll": 2.2074070007276267,
          "ece": 0.3872811547690316,
          "mean_confidence": 0.22271884523096844,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 98,
          "coverage_at_error_budget": 0.24,
          "risk_at_selected_coverage": 0.041666666666666664,
          "confidence_threshold_at_error_budget": 0.3036047699922756,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -26.0
        }
      },
      "pilot_attempted": 300
    },
    "gliner-decide": {
      "label": "GLiNER Decide 340M",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 47,
          "jev_correct": 48,
          "local_accuracy": 0.9791666666666666,
          "jev_accuracy": 1.0,
          "difference_pp": -2.083333333333333,
          "paired_cluster_bootstrap95_pp": [
            -6.25,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 1,
          "both_correct": 47,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 48,
          "jev_correct": 71,
          "local_accuracy": 0.6666666666666666,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -31.944444444444443,
          "paired_cluster_bootstrap95_pp": [
            -44.44444444444444,
            -19.444444444444443
          ],
          "local_only_correct": 1,
          "jev_only_correct": 24,
          "both_correct": 47,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 36,
          "jev_correct": 81,
          "local_accuracy": 0.32432432432432434,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -40.54054054054054,
          "paired_cluster_bootstrap95_pp": [
            -50.450450450450454,
            -30.63063063063063
          ],
          "local_only_correct": 3,
          "jev_only_correct": 48,
          "both_correct": 33,
          "both_wrong": 27,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 131,
          "jev_correct": 200,
          "local_accuracy": 0.5670995670995671,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -29.87012987012987,
          "paired_cluster_bootstrap95_pp": [
            -36.36363636363637,
            -23.376623376623378
          ],
          "local_only_correct": 4,
          "jev_only_correct": 73,
          "both_correct": 127,
          "both_wrong": 27,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.87,
          "macro_f1": 0.8597705802968961,
          "brier": 0.24659310599226772,
          "nll": 0.4878201931154672,
          "ece": 0.08354357530718455,
          "mean_confidence": 0.8654997817463251,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 97,
          "coverage_at_error_budget": 0.49,
          "risk_at_selected_coverage": 0.04081632653061224,
          "confidence_threshold_at_error_budget": 0.9501607621588721,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -4.0000000000000036
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.56,
          "macro_f1": 0.5575194821518351,
          "brier": 0.6174855949858955,
          "nll": 1.3122682205687546,
          "ece": 0.1331607122015432,
          "mean_confidence": 0.4513809523333549,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 98,
          "coverage_at_error_budget": 0.0,
          "risk_at_selected_coverage": null,
          "confidence_threshold_at_error_budget": null,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 8.000000000000007
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.75,
          "macro_f1": 0.7232142857142857,
          "brier": 0.3886990823056272,
          "nll": 1.1457760257668432,
          "ece": 0.1053961321659074,
          "mean_confidence": 0.6634389086315754,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 99,
          "coverage_at_error_budget": 0.44,
          "risk_at_selected_coverage": 0.045454545454545456,
          "confidence_threshold_at_error_budget": 0.8369045972261516,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -12.0
        }
      },
      "pilot_attempted": 300
    },
    "imajev-2b": {
      "label": "Imajev 2B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 47,
          "jev_correct": 48,
          "local_accuracy": 0.9791666666666666,
          "jev_accuracy": 1.0,
          "difference_pp": -2.083333333333333,
          "paired_cluster_bootstrap95_pp": [
            -6.25,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 1,
          "both_correct": 47,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 1
        },
        "original": {
          "n": 72,
          "local_correct": 65,
          "jev_correct": 71,
          "local_accuracy": 0.9027777777777778,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -8.333333333333332,
          "paired_cluster_bootstrap95_pp": [
            -16.666666666666668,
            -2.7777777777777777
          ],
          "local_only_correct": 0,
          "jev_only_correct": 6,
          "both_correct": 65,
          "both_wrong": 1,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 1
        },
        "hard": {
          "n": 111,
          "local_correct": 62,
          "jev_correct": 81,
          "local_accuracy": 0.5585585585585585,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -17.117117117117118,
          "paired_cluster_bootstrap95_pp": [
            -27.027027027027028,
            -7.207207207207207
          ],
          "local_only_correct": 7,
          "jev_only_correct": 26,
          "both_correct": 55,
          "both_wrong": 23,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 2
        },
        "all": {
          "n": 231,
          "local_correct": 174,
          "jev_correct": 200,
          "local_accuracy": 0.7532467532467533,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -11.255411255411255,
          "paired_cluster_bootstrap95_pp": [
            -16.883116883116884,
            -6.0606060606060606
          ],
          "local_only_correct": 7,
          "jev_only_correct": 33,
          "both_correct": 167,
          "both_wrong": 24,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 4
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.83,
          "macro_f1": 0.7994045451350954,
          "brier": 0.25723595129610055,
          "nll": 0.46473177507786134,
          "ece": 0.06853458021898708,
          "mean_confidence": 0.8486178378023093,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.61,
          "risk_at_selected_coverage": 0.04918032786885246,
          "confidence_threshold_at_error_budget": 0.8459627304966483,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -8.000000000000007
        },
        "emotiondair": {
          "n": 100,
          "valid": 86,
          "failures": 14,
          "accuracy": 0.45,
          "macro_f1": 0.4710320550499915,
          "brier": 0.6105742723102147,
          "nll": 1.2546197526322571,
          "ece": 0.1567660772005984,
          "mean_confidence": 0.6648142706388436,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.05,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9412578671122719,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -2.9999999999999973
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.77,
          "macro_f1": 0.7255291005291005,
          "brier": 0.6524168891871781,
          "nll": 1.8501214625634341,
          "ece": 0.5275087926273794,
          "mean_confidence": 0.24249120737262073,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.42,
          "risk_at_selected_coverage": 0.047619047619047616,
          "confidence_threshold_at_error_budget": 0.22137289716662784,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -9.999999999999998
        }
      },
      "pilot_attempted": 300
    },
    "imajev-4b": {
      "label": "Imajev 4B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 70,
          "jev_correct": 71,
          "local_accuracy": 0.9722222222222222,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -1.3888888888888888,
          "paired_cluster_bootstrap95_pp": [
            -4.166666666666667,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 1,
          "both_correct": 70,
          "both_wrong": 1,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 1
        },
        "hard": {
          "n": 111,
          "local_correct": 80,
          "jev_correct": 81,
          "local_accuracy": 0.7207207207207207,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -0.9009009009009009,
          "paired_cluster_bootstrap95_pp": [
            -7.207207207207207,
            5.405405405405405
          ],
          "local_only_correct": 6,
          "jev_only_correct": 7,
          "both_correct": 74,
          "both_wrong": 24,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 198,
          "jev_correct": 200,
          "local_accuracy": 0.8571428571428571,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -0.8658008658008658,
          "paired_cluster_bootstrap95_pp": [
            -3.896103896103896,
            2.1645021645021645
          ],
          "local_only_correct": 6,
          "jev_only_correct": 8,
          "both_correct": 192,
          "both_wrong": 25,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 1
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.86,
          "macro_f1": 0.8412217052546066,
          "brier": 0.17856040127088996,
          "nll": 0.31307796682724165,
          "ece": 0.08655937364313249,
          "mean_confidence": 0.8895075902446615,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.83,
          "risk_at_selected_coverage": 0.04819277108433735,
          "confidence_threshold_at_error_budget": 0.7958383706830272,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -5.000000000000004
        },
        "emotiondair": {
          "n": 100,
          "valid": 61,
          "failures": 39,
          "accuracy": 0.36,
          "macro_f1": 0.41241085806303196,
          "brier": 0.6312804192966041,
          "nll": 1.371791450805497,
          "ece": 0.26340717866645413,
          "mean_confidence": 0.8140283367353073,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.07,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9854528964433489,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -12.0
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.88,
          "macro_f1": 0.849537037037037,
          "brier": 0.14029582682211142,
          "nll": 0.2815529946286752,
          "ece": 0.05850419137217859,
          "mean_confidence": 0.9143536976855874,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.9,
          "risk_at_selected_coverage": 0.044444444444444446,
          "confidence_threshold_at_error_budget": 0.6506678469283724,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": 1.0000000000000009
        }
      },
      "pilot_attempted": 300
    },
    "intern-decision-4b": {
      "label": "Intern-Decision 4B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 71,
          "jev_correct": 71,
          "local_accuracy": 0.9861111111111112,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            -4.166666666666667,
            4.166666666666667
          ],
          "local_only_correct": 1,
          "jev_only_correct": 1,
          "both_correct": 70,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 82,
          "jev_correct": 81,
          "local_accuracy": 0.7387387387387387,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": 0.9009009009009009,
          "paired_cluster_bootstrap95_pp": [
            -8.108108108108109,
            9.90990990990991
          ],
          "local_only_correct": 12,
          "jev_only_correct": 11,
          "both_correct": 70,
          "both_wrong": 18,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 201,
          "jev_correct": 200,
          "local_accuracy": 0.8701298701298701,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": 0.4329004329004329,
          "paired_cluster_bootstrap95_pp": [
            -3.463203463203463,
            4.761904761904762
          ],
          "local_only_correct": 13,
          "jev_only_correct": 12,
          "both_correct": 188,
          "both_wrong": 18,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.92,
          "macro_f1": 0.9164432034853789,
          "brier": 0.11252898303347125,
          "nll": 0.25011118477446337,
          "ece": 0.09700771473600957,
          "mean_confidence": 0.8412572283353138,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.95,
          "risk_at_selected_coverage": 0.042105263157894736,
          "confidence_threshold_at_error_budget": 0.5657191264272909,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": 1.0000000000000009
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.47,
          "macro_f1": 0.4538709150154488,
          "brier": 0.7249356080664042,
          "nll": 1.462244028470997,
          "ece": 0.2511173483965282,
          "mean_confidence": 0.6685598295762987,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.04,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9584316482543075,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -1.0000000000000009
        },
        "banking77": {
          "n": 100,
          "valid": 0,
          "failures": 100,
          "accuracy": 0.0,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -87.0
        }
      },
      "pilot_attempted": 300
    },
    "jev-omni-12b-nf4": {
      "label": "Jev-Omni 12B NF4 (local variant)",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 69,
          "jev_correct": 71,
          "local_accuracy": 0.9583333333333334,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -2.7777777777777777,
          "paired_cluster_bootstrap95_pp": [
            -6.944444444444445,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 2,
          "both_correct": 69,
          "both_wrong": 1,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 81,
          "jev_correct": 81,
          "local_accuracy": 0.7297297297297297,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            -8.108108108108109,
            8.108108108108109
          ],
          "local_only_correct": 10,
          "jev_only_correct": 10,
          "both_correct": 71,
          "both_wrong": 20,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 198,
          "jev_correct": 200,
          "local_accuracy": 0.8571428571428571,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -0.8658008658008658,
          "paired_cluster_bootstrap95_pp": [
            -4.761904761904762,
            3.0303030303030303
          ],
          "local_only_correct": 10,
          "jev_only_correct": 12,
          "both_correct": 188,
          "both_wrong": 21,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.85,
          "macro_f1": 0.8287222714634879,
          "brier": 0.27866020465237795,
          "nll": 0.7507602014838137,
          "ece": 0.1305727425387731,
          "mean_confidence": 0.9624847637308229,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 96,
          "coverage_at_error_budget": 0.42,
          "risk_at_selected_coverage": 0.047619047619047616,
          "confidence_threshold_at_error_budget": 0.9981982739482542,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -6.000000000000005
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.46,
          "macro_f1": 0.4565447097705162,
          "brier": 0.9080267525397256,
          "nll": 2.5840585547838897,
          "ece": 0.4104201649243479,
          "mean_confidence": 0.8572782403855115,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 97,
          "coverage_at_error_budget": 0.05,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.998154663112092,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -1.9999999999999962
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.52,
          "macro_f1": 0.35551146384479715,
          "brier": 0.625188655171529,
          "nll": 2.6726959140532314,
          "ece": 0.1650498029346929,
          "mean_confidence": 0.6326123940155458,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 99,
          "coverage_at_error_budget": 0.08,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9918941470555838,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -35.0
        }
      },
      "pilot_attempted": 300
    },
    "jevk5-2b": {
      "label": "JevK5 2B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 57,
          "jev_correct": 71,
          "local_accuracy": 0.7916666666666666,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -19.444444444444446,
          "paired_cluster_bootstrap95_pp": [
            -30.555555555555557,
            -8.333333333333334
          ],
          "local_only_correct": 0,
          "jev_only_correct": 14,
          "both_correct": 57,
          "both_wrong": 1,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 68,
          "jev_correct": 81,
          "local_accuracy": 0.6126126126126126,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -11.711711711711711,
          "paired_cluster_bootstrap95_pp": [
            -21.62162162162162,
            -1.8018018018018018
          ],
          "local_only_correct": 9,
          "jev_only_correct": 22,
          "both_correct": 59,
          "both_wrong": 21,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 173,
          "jev_correct": 200,
          "local_accuracy": 0.7489177489177489,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -11.688311688311687,
          "paired_cluster_bootstrap95_pp": [
            -17.748917748917748,
            -6.0606060606060606
          ],
          "local_only_correct": 9,
          "jev_only_correct": 36,
          "both_correct": 164,
          "both_wrong": 22,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.85,
          "macro_f1": 0.8314418942816768,
          "brier": 0.20576121479385642,
          "nll": 0.35920588800342046,
          "ece": 0.03998630612186802,
          "mean_confidence": 0.8718456087506326,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 99,
          "coverage_at_error_budget": 0.73,
          "risk_at_selected_coverage": 0.0410958904109589,
          "confidence_threshold_at_error_budget": 0.8253642947426598,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -6.000000000000005
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.46,
          "macro_f1": 0.4620039080589347,
          "brier": 0.662964727960183,
          "nll": 1.361674546153259,
          "ece": 0.11456441974783631,
          "mean_confidence": 0.5745644197478363,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 99,
          "coverage_at_error_budget": 0.0,
          "risk_at_selected_coverage": null,
          "confidence_threshold_at_error_budget": null,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -1.9999999999999962
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.61,
          "macro_f1": 0.574074074074074,
          "brier": 0.5009899565734254,
          "nll": 1.3877493886647299,
          "ece": 0.09995903666884727,
          "mean_confidence": 0.5952308593142991,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.4,
          "risk_at_selected_coverage": 0.05,
          "confidence_threshold_at_error_budget": 0.6806439832850271,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -26.0
        }
      },
      "pilot_attempted": 300
    },
    "jevk5": {
      "label": "JevK5 4B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 68,
          "jev_correct": 71,
          "local_accuracy": 0.9444444444444444,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -4.166666666666666,
          "paired_cluster_bootstrap95_pp": [
            -12.5,
            2.7777777777777777
          ],
          "local_only_correct": 1,
          "jev_only_correct": 4,
          "both_correct": 67,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 86,
          "jev_correct": 81,
          "local_accuracy": 0.7747747747747747,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": 4.504504504504505,
          "paired_cluster_bootstrap95_pp": [
            -2.7027027027027026,
            11.711711711711711
          ],
          "local_only_correct": 12,
          "jev_only_correct": 7,
          "both_correct": 74,
          "both_wrong": 18,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 202,
          "jev_correct": 200,
          "local_accuracy": 0.8744588744588745,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": 0.8658008658008658,
          "paired_cluster_bootstrap95_pp": [
            -3.463203463203463,
            4.761904761904762
          ],
          "local_only_correct": 13,
          "jev_only_correct": 11,
          "both_correct": 189,
          "both_wrong": 18,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.86,
          "macro_f1": 0.8431784651649648,
          "brier": 0.18308041578904052,
          "nll": 0.3105711298700485,
          "ece": 0.07913129270570958,
          "mean_confidence": 0.913094446514173,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 100,
          "coverage_at_error_budget": 0.82,
          "risk_at_selected_coverage": 0.04878048780487805,
          "confidence_threshold_at_error_budget": 0.8103585145284068,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -5.000000000000004
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.49,
          "macro_f1": 0.47032227032227025,
          "brier": 0.6823550161267901,
          "nll": 1.4802894813664198,
          "ece": 0.17071480954222976,
          "mean_confidence": 0.6529049982180016,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 100,
          "coverage_at_error_budget": 0.06,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9714300682301723,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 1.0000000000000009
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.76,
          "macro_f1": 0.7421296296296297,
          "brier": 0.3539115558245326,
          "nll": 0.8811095231586809,
          "ece": 0.0785433485733122,
          "mean_confidence": 0.7320250221735181,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.5,
          "risk_at_selected_coverage": 0.04,
          "confidence_threshold_at_error_budget": 0.7892785800335825,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -10.999999999999998
        }
      },
      "pilot_attempted": 300
    },
    "julia": {
      "label": "Julia 144M",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 35,
          "jev_correct": 48,
          "local_accuracy": 0.7291666666666666,
          "jev_accuracy": 1.0,
          "difference_pp": -27.083333333333332,
          "paired_cluster_bootstrap95_pp": [
            -39.583333333333336,
            -14.583333333333334
          ],
          "local_only_correct": 0,
          "jev_only_correct": 13,
          "both_correct": 35,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 34,
          "jev_correct": 71,
          "local_accuracy": 0.4722222222222222,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -51.388888888888886,
          "paired_cluster_bootstrap95_pp": [
            -66.66666666666667,
            -36.111111111111114
          ],
          "local_only_correct": 1,
          "jev_only_correct": 38,
          "both_correct": 33,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 40,
          "jev_correct": 81,
          "local_accuracy": 0.36036036036036034,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -36.93693693693694,
          "paired_cluster_bootstrap95_pp": [
            -48.648648648648646,
            -24.324324324324323
          ],
          "local_only_correct": 10,
          "jev_only_correct": 51,
          "both_correct": 30,
          "both_wrong": 20,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 109,
          "jev_correct": 200,
          "local_accuracy": 0.47186147186147187,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -39.39393939393939,
          "paired_cluster_bootstrap95_pp": [
            -46.76406926406926,
            -31.6017316017316
          ],
          "local_only_correct": 11,
          "jev_only_correct": 102,
          "both_correct": 98,
          "both_wrong": 20,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.94,
          "macro_f1": 0.9399173646545361,
          "brier": 0.0858305675150916,
          "nll": 0.2301442574801705,
          "ece": 0.02596424910508795,
          "mean_confidence": 0.9541998074212948,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.98,
          "risk_at_selected_coverage": 0.04081632653061224,
          "confidence_threshold_at_error_budget": 0.5141316679414162,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": 2.9999999999999916
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.86,
          "macro_f1": 0.8556934538891329,
          "brier": 0.2529884129999399,
          "nll": 0.9901523885297542,
          "ece": 0.12537750112833054,
          "mean_confidence": 0.9853775011283306,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.75,
          "risk_at_selected_coverage": 0.04,
          "confidence_threshold_at_error_budget": 0.9999343456752893,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 38.0
        },
        "banking77": {
          "n": 100,
          "valid": 0,
          "failures": 100,
          "accuracy": 0.0,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -87.0
        }
      },
      "pilot_attempted": 300
    },
    "kev-08b": {
      "label": "Kev 0.8B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 58,
          "jev_correct": 71,
          "local_accuracy": 0.8055555555555556,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -18.055555555555554,
          "paired_cluster_bootstrap95_pp": [
            -30.555555555555557,
            -6.944444444444445
          ],
          "local_only_correct": 1,
          "jev_only_correct": 14,
          "both_correct": 57,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 41,
          "jev_correct": 81,
          "local_accuracy": 0.36936936936936937,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -36.03603603603604,
          "paired_cluster_bootstrap95_pp": [
            -45.968468468468465,
            -25.225225225225227
          ],
          "local_only_correct": 5,
          "jev_only_correct": 45,
          "both_correct": 36,
          "both_wrong": 25,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 147,
          "jev_correct": 200,
          "local_accuracy": 0.6363636363636364,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -22.943722943722943,
          "paired_cluster_bootstrap95_pp": [
            -29.004329004329005,
            -16.883116883116884
          ],
          "local_only_correct": 6,
          "jev_only_correct": 59,
          "both_correct": 141,
          "both_wrong": 25,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.92,
          "macro_f1": 0.9191676670668267,
          "brier": 0.13157860079529732,
          "nll": 0.262174546347939,
          "ece": 0.0476284714044861,
          "mean_confidence": 0.9229263926907052,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 99,
          "coverage_at_error_budget": 0.89,
          "risk_at_selected_coverage": 0.0449438202247191,
          "confidence_threshold_at_error_budget": 0.8274796608963421,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": 1.0000000000000009
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.45,
          "macro_f1": 0.44685533420827533,
          "brier": 0.6855933703177066,
          "nll": 1.4014917209018936,
          "ece": 0.07922613375573653,
          "mean_confidence": 0.46911125687679767,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 94,
          "coverage_at_error_budget": 0.02,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9188029770361273,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -2.9999999999999973
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.87,
          "macro_f1": 0.8592592592592593,
          "brier": 0.3221044727659176,
          "nll": 0.8750841858510173,
          "ece": 0.2952608316142175,
          "mean_confidence": 0.5761870190430436,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 100,
          "coverage_at_error_budget": 0.73,
          "risk_at_selected_coverage": 0.0410958904109589,
          "confidence_threshold_at_error_budget": 0.44848656576347934,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": 0.0
        }
      },
      "pilot_attempted": 300
    },
    "kev-4b": {
      "label": "Kev 4B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 67,
          "jev_correct": 71,
          "local_accuracy": 0.9305555555555556,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -5.555555555555555,
          "paired_cluster_bootstrap95_pp": [
            -15.277777777777779,
            1.3888888888888888
          ],
          "local_only_correct": 1,
          "jev_only_correct": 5,
          "both_correct": 66,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 60,
          "jev_correct": 81,
          "local_accuracy": 0.5405405405405406,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -18.91891891891892,
          "paired_cluster_bootstrap95_pp": [
            -27.027027027027028,
            -10.81081081081081
          ],
          "local_only_correct": 3,
          "jev_only_correct": 24,
          "both_correct": 57,
          "both_wrong": 27,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 175,
          "jev_correct": 200,
          "local_accuracy": 0.7575757575757576,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -10.822510822510822,
          "paired_cluster_bootstrap95_pp": [
            -16.017316017316016,
            -6.0606060606060606
          ],
          "local_only_correct": 4,
          "jev_only_correct": 29,
          "both_correct": 171,
          "both_wrong": 27,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.95,
          "macro_f1": 0.9497288277012933,
          "brier": 0.07434302173297898,
          "nll": 0.15426421599175744,
          "ece": 0.04965625415241842,
          "mean_confidence": 0.9350875022936641,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 98,
          "coverage_at_error_budget": 1.0,
          "risk_at_selected_coverage": 0.05,
          "confidence_threshold_at_error_budget": 0.5382301221382434,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": 3.9999999999999925
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.45,
          "macro_f1": 0.4621142833601121,
          "brier": 0.7130170685271353,
          "nll": 1.5867646251948202,
          "ece": 0.173407318498611,
          "mean_confidence": 0.623407318498611,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 96,
          "coverage_at_error_budget": 0.04,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9668808133708311,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -2.9999999999999973
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.88,
          "macro_f1": 0.8657407407407407,
          "brier": 0.22561721256907089,
          "nll": 0.555370653831509,
          "ece": 0.18387685619675537,
          "mean_confidence": 0.6995450890803078,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 96,
          "coverage_at_error_budget": 0.86,
          "risk_at_selected_coverage": 0.046511627906976744,
          "confidence_threshold_at_error_budget": 0.44813875256001356,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": 1.0000000000000009
        }
      },
      "pilot_attempted": 300
    },
    "laya-multilingual": {
      "label": "Laya multilingual",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 43,
          "jev_correct": 48,
          "local_accuracy": 0.8958333333333334,
          "jev_accuracy": 1.0,
          "difference_pp": -10.416666666666668,
          "paired_cluster_bootstrap95_pp": [
            -18.75,
            -2.0833333333333335
          ],
          "local_only_correct": 0,
          "jev_only_correct": 5,
          "both_correct": 43,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 29,
          "jev_correct": 71,
          "local_accuracy": 0.4027777777777778,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -58.333333333333336,
          "paired_cluster_bootstrap95_pp": [
            -70.83333333333333,
            -45.833333333333336
          ],
          "local_only_correct": 0,
          "jev_only_correct": 42,
          "both_correct": 29,
          "both_wrong": 1,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 37,
          "jev_correct": 81,
          "local_accuracy": 0.3333333333333333,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -39.63963963963964,
          "paired_cluster_bootstrap95_pp": [
            -50.450450450450454,
            -27.92792792792793
          ],
          "local_only_correct": 10,
          "jev_only_correct": 54,
          "both_correct": 27,
          "both_wrong": 20,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 109,
          "jev_correct": 200,
          "local_accuracy": 0.47186147186147187,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -39.39393939393939,
          "paired_cluster_bootstrap95_pp": [
            -45.887445887445885,
            -32.03463203463203
          ],
          "local_only_correct": 10,
          "jev_only_correct": 101,
          "both_correct": 99,
          "both_wrong": 21,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.91,
          "macro_f1": 0.9088293058630981,
          "brier": 0.2237508917900563,
          "nll": 0.46348664195762923,
          "ece": 0.2134342822947028,
          "mean_confidence": 0.7020402651600428,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 38,
          "coverage_at_error_budget": 0.82,
          "risk_at_selected_coverage": 0.04878048780487805,
          "confidence_threshold_at_error_budget": 0.5072,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": 0.0
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.22,
          "macro_f1": 0.16117355007648235,
          "brier": 0.861136964113494,
          "nll": 1.8788848234916093,
          "ece": 0.2193491218774652,
          "mean_confidence": 0.4359327802433019,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 33,
          "coverage_at_error_budget": 0.01,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.8291829182918292,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -26.0
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.02,
          "macro_f1": 0.0005446623093681917,
          "brier": 1.0169471690482537,
          "nll": 4.3910137279690895,
          "ece": 0.13938411458499564,
          "mean_confidence": 0.15938411458499566,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 96,
          "coverage_at_error_budget": 0.0,
          "risk_at_selected_coverage": null,
          "confidence_threshold_at_error_budget": null,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -85.0
        }
      },
      "pilot_attempted": 300
    },
    "laya": {
      "label": "Laya",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 46,
          "jev_correct": 48,
          "local_accuracy": 0.9583333333333334,
          "jev_accuracy": 1.0,
          "difference_pp": -4.166666666666666,
          "paired_cluster_bootstrap95_pp": [
            -10.416666666666666,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 2,
          "both_correct": 46,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 52,
          "jev_correct": 71,
          "local_accuracy": 0.7222222222222222,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -26.38888888888889,
          "paired_cluster_bootstrap95_pp": [
            -40.27777777777778,
            -13.88888888888889
          ],
          "local_only_correct": 1,
          "jev_only_correct": 20,
          "both_correct": 51,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 37,
          "jev_correct": 81,
          "local_accuracy": 0.3333333333333333,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -39.63963963963964,
          "paired_cluster_bootstrap95_pp": [
            -52.252252252252255,
            -27.027027027027028
          ],
          "local_only_correct": 10,
          "jev_only_correct": 54,
          "both_correct": 27,
          "both_wrong": 20,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 135,
          "jev_correct": 200,
          "local_accuracy": 0.5844155844155844,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -28.13852813852814,
          "paired_cluster_bootstrap95_pp": [
            -35.064935064935064,
            -20.77922077922078
          ],
          "local_only_correct": 11,
          "jev_only_correct": 76,
          "both_correct": 124,
          "both_wrong": 20,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.97,
          "macro_f1": 0.9699754901960784,
          "brier": 0.09941611358150551,
          "nll": 0.2690843957108369,
          "ece": 0.14989857344790564,
          "mean_confidence": 0.8201014265520942,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 35,
          "coverage_at_error_budget": 1.0,
          "risk_at_selected_coverage": 0.03,
          "confidence_threshold_at_error_budget": 0.4669466946694669,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": 5.999999999999995
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.4,
          "macro_f1": 0.37363383532848143,
          "brier": 0.9615488315974042,
          "nll": 2.426368351769062,
          "ece": 0.4130517286398314,
          "mean_confidence": 0.8034912133883063,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 51,
          "coverage_at_error_budget": 0.06,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9963003699630036,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -7.9999999999999964
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.02,
          "macro_f1": 0.0006858710562414266,
          "brier": 1.001554764081172,
          "nll": 4.8139742290460665,
          "ece": 0.08644022339654749,
          "mean_confidence": 0.10644022339654746,
          "true_label_zero_rate": 0.01,
          "renormalized_vectors": 86,
          "coverage_at_error_budget": 0.0,
          "risk_at_selected_coverage": null,
          "confidence_threshold_at_error_budget": null,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -85.0
        }
      },
      "pilot_attempted": 300
    },
    "nev-2b": {
      "label": "Nev 2B Q8",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 63,
          "jev_correct": 71,
          "local_accuracy": 0.875,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -11.11111111111111,
          "paired_cluster_bootstrap95_pp": [
            -19.444444444444443,
            -4.166666666666667
          ],
          "local_only_correct": 1,
          "jev_only_correct": 9,
          "both_correct": 62,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 75,
          "jev_correct": 81,
          "local_accuracy": 0.6756756756756757,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -5.405405405405405,
          "paired_cluster_bootstrap95_pp": [
            -14.414414414414415,
            3.6036036036036037
          ],
          "local_only_correct": 10,
          "jev_only_correct": 16,
          "both_correct": 65,
          "both_wrong": 20,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 186,
          "jev_correct": 200,
          "local_accuracy": 0.8051948051948052,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -6.0606060606060606,
          "paired_cluster_bootstrap95_pp": [
            -10.822510822510823,
            -1.2987012987012987
          ],
          "local_only_correct": 11,
          "jev_only_correct": 25,
          "both_correct": 175,
          "both_wrong": 20,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.88,
          "macro_f1": 0.8702328101051742,
          "brier": 0.16630003993102566,
          "nll": 0.30537952434188215,
          "ece": 0.04927288971104314,
          "mean_confidence": 0.8868288770170052,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.72,
          "risk_at_selected_coverage": 0.041666666666666664,
          "confidence_threshold_at_error_budget": 0.8746545770002409,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -3.0000000000000027
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.48,
          "macro_f1": 0.4983502512812858,
          "brier": 0.7818009747578981,
          "nll": 1.5889221916350404,
          "ece": 0.30378130375731727,
          "mean_confidence": 0.75494268047813,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.02,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9832776362975374,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 0.0
        },
        "banking77": {
          "n": 100,
          "valid": 0,
          "failures": 100,
          "accuracy": 0.0,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -87.0
        }
      },
      "pilot_attempted": 300
    },
    "nimble-9b": {
      "label": "Nimble 9B NF4",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 68,
          "jev_correct": 71,
          "local_accuracy": 0.9444444444444444,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -4.166666666666666,
          "paired_cluster_bootstrap95_pp": [
            -13.88888888888889,
            2.7777777777777777
          ],
          "local_only_correct": 1,
          "jev_only_correct": 4,
          "both_correct": 67,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 65,
          "jev_correct": 81,
          "local_accuracy": 0.5855855855855856,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -14.414414414414415,
          "paired_cluster_bootstrap95_pp": [
            -22.52252252252252,
            -6.306306306306307
          ],
          "local_only_correct": 4,
          "jev_only_correct": 20,
          "both_correct": 61,
          "both_wrong": 26,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 181,
          "jev_correct": 200,
          "local_accuracy": 0.7835497835497836,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -8.225108225108226,
          "paired_cluster_bootstrap95_pp": [
            -13.41991341991342,
            -3.463203463203463
          ],
          "local_only_correct": 5,
          "jev_only_correct": 24,
          "both_correct": 176,
          "both_wrong": 26,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.94,
          "macro_f1": 0.9393627450980392,
          "brier": 0.10161924818545383,
          "nll": 0.21588406841713834,
          "ece": 0.0190632077541467,
          "mean_confidence": 0.9237485182505192,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.95,
          "risk_at_selected_coverage": 0.042105263157894736,
          "confidence_threshold_at_error_budget": 0.5904498919850004,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": 2.9999999999999916
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.45,
          "macro_f1": 0.4431908821522525,
          "brier": 0.8268843824349763,
          "nll": 1.8682592083733067,
          "ece": 0.3434679904627447,
          "mean_confidence": 0.7934679904627447,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.01,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9984524133937345,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -2.9999999999999973
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.84,
          "macro_f1": 0.8342592592592593,
          "brier": 0.22321636863385308,
          "nll": 0.5573742691498393,
          "ece": 0.0570569823355788,
          "mean_confidence": 0.8782270042496466,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.8,
          "risk_at_selected_coverage": 0.05,
          "confidence_threshold_at_error_budget": 0.7829002475214734,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -3.0000000000000027
        }
      },
      "pilot_attempted": 300
    },
    "plumb-4b": {
      "label": "Plumb 4B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 70,
          "jev_correct": 71,
          "local_accuracy": 0.9722222222222222,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -1.3888888888888888,
          "paired_cluster_bootstrap95_pp": [
            -6.944444444444445,
            2.7777777777777777
          ],
          "local_only_correct": 1,
          "jev_only_correct": 2,
          "both_correct": 69,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 89,
          "jev_correct": 81,
          "local_accuracy": 0.8018018018018018,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": 7.207207207207207,
          "paired_cluster_bootstrap95_pp": [
            0.8783783783783796,
            14.414414414414415
          ],
          "local_only_correct": 12,
          "jev_only_correct": 4,
          "both_correct": 77,
          "both_wrong": 18,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 207,
          "jev_correct": 200,
          "local_accuracy": 0.8961038961038961,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": 3.0303030303030303,
          "paired_cluster_bootstrap95_pp": [
            -0.4329004329004329,
            6.926406926406926
          ],
          "local_only_correct": 13,
          "jev_only_correct": 6,
          "both_correct": 194,
          "both_wrong": 18,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.89,
          "macro_f1": 0.8804870754667915,
          "brier": 0.17050518994493633,
          "nll": 0.33700076520730404,
          "ece": 0.12610280148022252,
          "mean_confidence": 0.8553880452149274,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 99,
          "coverage_at_error_budget": 0.82,
          "risk_at_selected_coverage": 0.04878048780487805,
          "confidence_threshold_at_error_budget": 0.7676754937026871,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -2.0000000000000018
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.47,
          "macro_f1": 0.47823857204094294,
          "brier": 0.6883173471889722,
          "nll": 1.4181417680716755,
          "ece": 0.17751330564463857,
          "mean_confidence": 0.6414588244157613,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 96,
          "coverage_at_error_budget": 0.06,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.8750845665448143,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -1.0000000000000009
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.77,
          "macro_f1": 0.7330026455026455,
          "brier": 0.3457448184028046,
          "nll": 0.9003317047307804,
          "ece": 0.08702095162910585,
          "mean_confidence": 0.7539331276512717,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.44,
          "risk_at_selected_coverage": 0.045454545454545456,
          "confidence_threshold_at_error_budget": 0.8030619196794472,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -9.999999999999998
        }
      },
      "pilot_attempted": 300
    },
    "tev1": {
      "label": "Tev1 4B",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 69,
          "jev_correct": 71,
          "local_accuracy": 0.9583333333333334,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -2.7777777777777777,
          "paired_cluster_bootstrap95_pp": [
            -9.722222222222221,
            2.7777777777777777
          ],
          "local_only_correct": 1,
          "jev_only_correct": 3,
          "both_correct": 68,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 57,
          "jev_correct": 81,
          "local_accuracy": 0.5135135135135135,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -21.62162162162162,
          "paired_cluster_bootstrap95_pp": [
            -29.72972972972973,
            -13.513513513513514
          ],
          "local_only_correct": 3,
          "jev_only_correct": 27,
          "both_correct": 54,
          "both_wrong": 27,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 174,
          "jev_correct": 200,
          "local_accuracy": 0.7532467532467533,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -11.255411255411255,
          "paired_cluster_bootstrap95_pp": [
            -16.450216450216452,
            -6.926406926406926
          ],
          "local_only_correct": 4,
          "jev_only_correct": 30,
          "both_correct": 170,
          "both_wrong": 27,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.96,
          "macro_f1": 0.9597919167667067,
          "brier": 0.07563144258905968,
          "nll": 0.18157610706035143,
          "ece": 0.04953170097838836,
          "mean_confidence": 0.9104682990216117,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 98,
          "coverage_at_error_budget": 1.0,
          "risk_at_selected_coverage": 0.04,
          "confidence_threshold_at_error_budget": 0.49112449624277393,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": 4.999999999999993
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.48,
          "macro_f1": 0.4692623057840448,
          "brier": 0.7389182599470188,
          "nll": 1.833062557056154,
          "ece": 0.22909298412548756,
          "mean_confidence": 0.7090929841254874,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 94,
          "coverage_at_error_budget": 0.04,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9925325520396537,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 0.0
        },
        "banking77": {
          "n": 100,
          "valid": 0,
          "failures": 100,
          "accuracy": 0.0,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -87.0
        }
      },
      "pilot_attempted": 300
    },
    "von": {
      "label": "Von",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 46,
          "jev_correct": 71,
          "local_accuracy": 0.6388888888888888,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -34.72222222222222,
          "paired_cluster_bootstrap95_pp": [
            -50.0,
            -20.833333333333332
          ],
          "local_only_correct": 1,
          "jev_only_correct": 26,
          "both_correct": 45,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 50,
          "jev_correct": 81,
          "local_accuracy": 0.45045045045045046,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -27.927927927927925,
          "paired_cluster_bootstrap95_pp": [
            -38.73873873873874,
            -16.216216216216218
          ],
          "local_only_correct": 8,
          "jev_only_correct": 39,
          "both_correct": 42,
          "both_wrong": 22,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 144,
          "jev_correct": 200,
          "local_accuracy": 0.6233766233766234,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -24.242424242424242,
          "paired_cluster_bootstrap95_pp": [
            -31.16883116883117,
            -16.883116883116884
          ],
          "local_only_correct": 9,
          "jev_only_correct": 65,
          "both_correct": 135,
          "both_wrong": 22,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.89,
          "macro_f1": 0.8902017435452085,
          "brier": 0.18437170670952174,
          "nll": 0.3955986578235469,
          "ece": 0.13324820469523213,
          "mean_confidence": 0.7567517953047681,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 21,
          "coverage_at_error_budget": 0.82,
          "risk_at_selected_coverage": 0.04878048780487805,
          "confidence_threshold_at_error_budget": 0.6179,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -2.0000000000000018
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.71,
          "macro_f1": 0.679746342981637,
          "brier": 0.3660230881474725,
          "nll": 0.6874096106175935,
          "ece": 0.07978573491765012,
          "mean_confidence": 0.6596374052887612,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 49,
          "coverage_at_error_budget": 0.4,
          "risk_at_selected_coverage": 0.05,
          "confidence_threshold_at_error_budget": 0.7184718471847185,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 23.0
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.78,
          "macro_f1": 0.7572089947089947,
          "brier": 0.35561507972883205,
          "nll": 1.926319450523956,
          "ece": 0.15217235951997354,
          "mean_confidence": 0.9045831182382612,
          "true_label_zero_rate": 0.05,
          "renormalized_vectors": 16,
          "coverage_at_error_budget": 0.42,
          "risk_at_selected_coverage": 0.047619047619047616,
          "confidence_threshold_at_error_budget": 0.9997999799979999,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -8.999999999999996
        }
      },
      "pilot_attempted": 300
    },
    "wald-4b-v12": {
      "label": "Wald 4B v1.2",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 72,
          "jev_correct": 71,
          "local_accuracy": 1.0,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": 1.3888888888888888,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            4.166666666666667
          ],
          "local_only_correct": 1,
          "jev_only_correct": 0,
          "both_correct": 71,
          "both_wrong": 0,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 77,
          "jev_correct": 81,
          "local_accuracy": 0.6936936936936937,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": -3.6036036036036037,
          "paired_cluster_bootstrap95_pp": [
            -12.612612612612613,
            5.405405405405405
          ],
          "local_only_correct": 11,
          "jev_only_correct": 15,
          "both_correct": 66,
          "both_wrong": 19,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 10
        },
        "all": {
          "n": 231,
          "local_correct": 197,
          "jev_correct": 200,
          "local_accuracy": 0.8528138528138528,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -1.2987012987012987,
          "paired_cluster_bootstrap95_pp": [
            -5.627705627705628,
            3.0303030303030303
          ],
          "local_only_correct": 12,
          "jev_only_correct": 15,
          "both_correct": 185,
          "both_wrong": 19,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 10
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.83,
          "macro_f1": 0.8005719154442794,
          "brier": 0.22129112520857727,
          "nll": 0.3917590061284997,
          "ece": 0.08545949469373493,
          "mean_confidence": 0.8828661614231033,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.74,
          "risk_at_selected_coverage": 0.04054054054054054,
          "confidence_threshold_at_error_budget": 0.8587778928868025,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -8.000000000000007
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.43,
          "macro_f1": 0.43809222730791353,
          "brier": 0.7544256854742735,
          "nll": 1.513478498919591,
          "ece": 0.27220345126531353,
          "mean_confidence": 0.6836026476988465,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.02,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.972520312350378,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": -4.999999999999999
        },
        "banking77": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.84,
          "macro_f1": 0.8111111111111111,
          "brier": 0.24577066083771545,
          "nll": 0.52534408214467,
          "ece": 0.08558354442787915,
          "mean_confidence": 0.8332043830397136,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.7,
          "risk_at_selected_coverage": 0.04285714285714286,
          "confidence_threshold_at_error_budget": 0.7602499156485382,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -3.0000000000000027
        }
      },
      "pilot_attempted": 300
    },
    "winnow-12b-q8": {
      "label": "Winnow 12B Q8 (Windows CUDA)",
      "public_attempted": 231,
      "public_complete": true,
      "public": {
        "easy": {
          "n": 48,
          "local_correct": 48,
          "jev_correct": 48,
          "local_accuracy": 1.0,
          "jev_accuracy": 1.0,
          "difference_pp": 0.0,
          "paired_cluster_bootstrap95_pp": [
            0.0,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 0,
          "both_correct": 48,
          "both_wrong": 0,
          "independent_groups": 48,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "original": {
          "n": 72,
          "local_correct": 69,
          "jev_correct": 71,
          "local_accuracy": 0.9583333333333334,
          "jev_accuracy": 0.9861111111111112,
          "difference_pp": -2.7777777777777777,
          "paired_cluster_bootstrap95_pp": [
            -8.333333333333334,
            0.0
          ],
          "local_only_correct": 0,
          "jev_only_correct": 2,
          "both_correct": 69,
          "both_wrong": 1,
          "independent_groups": 36,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "hard": {
          "n": 111,
          "local_correct": 82,
          "jev_correct": 81,
          "local_accuracy": 0.7387387387387387,
          "jev_accuracy": 0.7297297297297297,
          "difference_pp": 0.9009009009009009,
          "paired_cluster_bootstrap95_pp": [
            -6.328828828828828,
            9.00900900900901
          ],
          "local_only_correct": 10,
          "jev_only_correct": 9,
          "both_correct": 72,
          "both_wrong": 20,
          "independent_groups": 111,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        },
        "all": {
          "n": 231,
          "local_correct": 199,
          "jev_correct": 200,
          "local_accuracy": 0.8614718614718615,
          "jev_accuracy": 0.8658008658008658,
          "difference_pp": -0.4329004329004329,
          "paired_cluster_bootstrap95_pp": [
            -4.761904761904762,
            3.463203463203463
          ],
          "local_only_correct": 10,
          "jev_only_correct": 11,
          "both_correct": 189,
          "both_wrong": 21,
          "independent_groups": 195,
          "interval_note": "Descriptive paired bootstrap, 2000 resamples, stratified by public tier and clustered by provided paraphrase group; not adjusted for multiple model comparisons.",
          "local_invalid": 0
        }
      },
      "pilot": {
        "agnews": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.88,
          "macro_f1": 0.8675169687777176,
          "brier": 0.19809081466567932,
          "nll": 0.3973559819715019,
          "ece": 0.07743457587796397,
          "mean_confidence": 0.9472118853560416,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.69,
          "risk_at_selected_coverage": 0.043478260869565216,
          "confidence_threshold_at_error_budget": 0.9824529150750267,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.91,
          "difference_pp": -3.0000000000000027
        },
        "emotiondair": {
          "n": 100,
          "valid": 100,
          "failures": 0,
          "accuracy": 0.52,
          "macro_f1": 0.5101097065702608,
          "brier": 0.8032170193394372,
          "nll": 2.288914917271393,
          "ece": 0.33328254780708955,
          "mean_confidence": 0.8532825478070898,
          "true_label_zero_rate": 0.0,
          "renormalized_vectors": 0,
          "coverage_at_error_budget": 0.04,
          "risk_at_selected_coverage": 0.0,
          "confidence_threshold_at_error_budget": 0.9995286729708804,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.48,
          "difference_pp": 4.0000000000000036
        },
        "banking77": {
          "n": 100,
          "valid": 0,
          "failures": 100,
          "accuracy": 0.0,
          "coverage_at_error_budget_note": "Threshold fitted on this test slice: retrospective diagnostic, not an automation guarantee.",
          "published_jev_accuracy": 0.87,
          "difference_pp": -87.0
        }
      },
      "pilot_attempted": 300
    }
  },
  "public_case_count": 231,
  "pilot_case_count": 300
}