{ "updated": "2026-10-05T00:49:41.843395+00:00", "scope": "231 public JevBench cases; original scoring. Separate from the sealed leaderboard and from full English/Norwegian evaluation.", "fixture_sha256": "8825e4e9785f4f34dbaa996d8b4d1e313d095d8eafc8f865c966914fed85c5a7", "models": { "decision2-kai-06b": { "label": "Decision 2.0 Kai 0.6B", "source": "https://huggingface.co/vllm-sr/Decision-2.0-Kai-0.6B", "revision": "d06cf74b1304c6454b9de6e0adb8eaad5ac719fa", "public231": { "expected": 231, "attempted": 231, "complete": true, "correct": 152, "errors": 0, "tiers": { "jevbench_public/easy": { "n": 48, "correct": 47, "accuracy": 0.9791666666666666, "strict_valid": 48, "valid": 48, "renormalized": 0 }, "jevbench_public/original": { "n": 72, "correct": 61, "accuracy": 0.8472222222222222, "strict_valid": 72, "valid": 72, "renormalized": 0 }, "jevbench_public/hard": { "n": 111, "correct": 44, "accuracy": 0.3963963963963964, "strict_valid": 111, "valid": 111, "renormalized": 0 } }, "accuracy": 0.658008658008658 }, "shared_suites": { "jev_verified": { "expected": 300, "attempted": 300, "evaluation_expected": 300, "suites": { "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "correct": 83, "accuracy": 0.83, "wilson95": [ 0.7445199523239887, 0.8910643388594006 ], "head_correct": 83, "heads": 100, "head_accuracy": 0.83, "coverage": 1.0, "accuracy_on_successful": 0.83, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8157979764410472, "answered_single_label_heads": 100, "ece10": 0.058847022430903076, "brier": 0.25782996402600833 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "head_correct": 53, "heads": 100, "head_accuracy": 0.53, "coverage": 1.0, "accuracy_on_successful": 0.53, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5344801676533546, "answered_single_label_heads": 100, "ece10": 0.10603260356947049, "brier": 0.6067726884898922 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "correct": 81, "accuracy": 0.81, "wilson95": [ 0.7222115462093562, 0.8748524849023126 ], "head_correct": 81, "heads": 100, "head_accuracy": 0.81, "coverage": 1.0, "accuracy_on_successful": 0.81, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8109126984126984, "answered_single_label_heads": 100, "ece10": 0.13960721702872478, "brier": 0.2573548559322787 } } }, "jev_fresh": { "expected": 1264, "attempted": 1264, "evaluation_expected": 848, "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "correct": 167, "accuracy": 0.835, "wilson95": [ 0.7773421846094045, 0.8800314467178268 ], "head_correct": 167, "heads": 200, "head_accuracy": 0.835, "coverage": 1.0, "accuracy_on_successful": 0.835, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8358293764514856, "answered_single_label_heads": 200, "ece10": 0.03423306873028258, "brier": 0.23876398953271255 }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "correct": 114, "accuracy": 0.57, "wilson95": [ 0.5007048776415268, 0.6366567766656557 ], "head_correct": 114, "heads": 200, "head_accuracy": 0.57, "coverage": 1.0, "accuracy_on_successful": 0.57, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.455123840196304, "answered_single_label_heads": 200, "ece10": 0.15121845892655822, "brier": 0.6009159997451833 }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "correct": 109, "accuracy": 0.545, "wilson95": [ 0.47578611372488105, 0.6125178069011651 ], "head_correct": 109, "heads": 200, "head_accuracy": 0.545, "coverage": 1.0, "accuracy_on_successful": 0.545, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.35275080906148865, "answered_single_label_heads": 200, "ece10": 0.14989966256286957, "brier": 0.5101996171268393 }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "correct": 137, "accuracy": 0.685, "wilson95": [ 0.6176504391426966, 0.745376790097715 ], "head_correct": 137, "heads": 200, "head_accuracy": 0.685, "coverage": 1.0, "accuracy_on_successful": 0.685, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6424416129856125, "answered_single_label_heads": 200, "ece10": 0.062413774053039606, "brier": 0.395779425742393 }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "correct": 7, "accuracy": 0.5833333333333334, "wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "head_correct": 7, "heads": 12, "head_accuracy": 0.5833333333333334, "coverage": 1.0, "accuracy_on_successful": 0.5833333333333334, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5771428571428572, "answered_single_label_heads": 12, "ece10": 0.1284240542796109, "brier": 0.47550364635634823 }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "correct": 2, "accuracy": 0.16666666666666666, "wilson95": [ 0.04696514218385378, 0.4480308622529734 ], "head_correct": 2, "heads": 12, "head_accuracy": 0.16666666666666666, "coverage": 1.0, "accuracy_on_successful": 0.16666666666666666, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.06666666666666667, "answered_single_label_heads": 12, "ece10": 0.2861870202220236, "brier": 0.8215948007133456 }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "head_correct": 11, "heads": 12, "head_accuracy": 0.9166666666666666, "coverage": 1.0, "accuracy_on_successful": 0.9166666666666666, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9142857142857144, "answered_single_label_heads": 12, "ece10": 0.26215566433068593, "brier": 0.22392829326097405 }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "head_correct": 11, "heads": 12, "head_accuracy": 0.9166666666666666, "coverage": 1.0, "accuracy_on_successful": 0.9166666666666666, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9142857142857144, "answered_single_label_heads": 12, "ece10": 0.24116465184115785, "brier": 0.15073132212552853 } } }, "alternatives": { "expected": 12341, "attempted": 12341, "evaluation_expected": 12341, "suites": { "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "head_correct": 35, "heads": 40, "head_accuracy": 0.875, "coverage": 1.0, "accuracy_on_successful": 0.875, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8721691672620466, "answered_single_label_heads": 40, "ece10": 0.2564489347600351, "brier": 0.2320485160391764 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "head_correct": 35, "heads": 40, "head_accuracy": 0.875, "coverage": 1.0, "accuracy_on_successful": 0.875, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8705368814192344, "answered_single_label_heads": 40, "ece10": 0.21250871140804717, "brier": 0.25180047609277456 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "correct": 31, "accuracy": 0.775, "wilson95": [ 0.6249690325768998, 0.8768390867645927 ], "head_correct": 31, "heads": 40, "head_accuracy": 0.775, "coverage": 1.0, "accuracy_on_successful": 0.775, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7758963585434173, "answered_single_label_heads": 40, "ece10": 0.19663876285841908, "brier": 0.4012655611093387 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "correct": 31, "accuracy": 0.775, "wilson95": [ 0.6249690325768998, 0.8768390867645927 ], "head_correct": 31, "heads": 40, "head_accuracy": 0.775, "coverage": 1.0, "accuracy_on_successful": 0.775, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7761904761904762, "answered_single_label_heads": 40, "ece10": 0.19300226398447437, "brier": 0.4008199407354482 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "head_correct": 28, "heads": 28, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 28, "ece10": 0.14128307420775635, "brier": 0.06380799107572212 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "head_correct": 28, "heads": 28, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 28, "ece10": 0.15364168812653817, "brier": 0.0726062278474047 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "correct": 26, "accuracy": 0.9285714285714286, "wilson95": [ 0.7735463743160095, 0.9801879358846942 ], "head_correct": 26, "heads": 28, "head_accuracy": 0.9285714285714286, "coverage": 1.0, "accuracy_on_successful": 0.9285714285714286, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9283882783882784, "answered_single_label_heads": 28, "ece10": 0.1336028324279874, "brier": 0.15032549297794265 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "head_correct": 27, "heads": 28, "head_accuracy": 0.9642857142857143, "coverage": 1.0, "accuracy_on_successful": 0.9642857142857143, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9641025641025641, "answered_single_label_heads": 28, "ece10": 0.18117743167152622, "brier": 0.15034880975738166 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "correct": 82, "accuracy": 0.82, "wilson95": [ 0.7333264161369926, 0.8829977450105364 ], "head_correct": 82, "heads": 100, "head_accuracy": 0.82, "coverage": 1.0, "accuracy_on_successful": 0.82, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8105873001777394, "answered_single_label_heads": 100, "ece10": 0.08607047621124106, "brier": 0.23487266591653483 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "head_correct": 53, "heads": 100, "head_accuracy": 0.53, "coverage": 1.0, "accuracy_on_successful": 0.53, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5405435251023486, "answered_single_label_heads": 100, "ece10": 0.1485330426219207, "brier": 0.6315653513399713 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "correct": 81, "accuracy": 0.81, "wilson95": [ 0.7222115462093562, 0.8748524849023126 ], "head_correct": 81, "heads": 100, "head_accuracy": 0.81, "coverage": 1.0, "accuracy_on_successful": 0.81, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.801653439153439, "answered_single_label_heads": 100, "ece10": 0.14713291084332586, "brier": 0.2582749820430068 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "head_correct": 58, "heads": 100, "head_accuracy": 0.58, "coverage": 1.0, "accuracy_on_successful": 0.58, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5798319327731092, "answered_single_label_heads": 100, "ece10": 0.025317954555911105, "brier": 0.48261393914135825 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "correct": 55, "accuracy": 0.55, "wilson95": [ 0.45244602997442135, 0.6438546202048803 ], "head_correct": 55, "heads": 100, "head_accuracy": 0.55, "coverage": 1.0, "accuracy_on_successful": 0.55, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5575070723909733, "answered_single_label_heads": 100, "ece10": 0.0857158519487958, "brier": 0.5841443020443857 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "head_correct": 142, "heads": 200, "head_accuracy": 0.71, "coverage": 1.0, "accuracy_on_successful": 0.49, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 200, "ece10": 0.10177851149843091, "brier": 0.38631272989323356 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "head_correct": 127, "heads": 200, "head_accuracy": 0.635, "coverage": 1.0, "accuracy_on_successful": 0.41, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 200, "ece10": 0.11902855094601517, "brier": 0.49505592715748564 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "head_correct": 80, "heads": 100, "head_accuracy": 0.8, "coverage": 1.0, "accuracy_on_successful": 0.8, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7480984607576288, "answered_single_label_heads": 100, "ece10": 0.10201729325744928, "brier": 0.3325461174603886 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "head_correct": 186, "heads": 400, "head_accuracy": 0.465, "coverage": 1.0, "accuracy_on_successful": 0.08, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 400, "ece10": 0.15512189403789176, "brier": 0.668042998120885 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "correct": 57, "accuracy": 0.57, "wilson95": [ 0.47215389549212305, 0.6626670147588988 ], "head_correct": 57, "heads": 100, "head_accuracy": 0.57, "coverage": 1.0, "accuracy_on_successful": 0.57, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5641940106826364, "answered_single_label_heads": 100, "ece10": 0.10447210768382052, "brier": 0.5870684300051682 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "head_correct": 42, "heads": 100, "head_accuracy": 0.42, "coverage": 1.0, "accuracy_on_successful": 0.42, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.3948316108052417, "answered_single_label_heads": 100, "ece10": 0.1287123507316319, "brier": 0.761071458148238 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "head_correct": 72, "heads": 200, "head_accuracy": 0.36, "coverage": 1.0, "accuracy_on_successful": 0.04, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.09227842871504895, "brier": 0.5391208850789708 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "correct": 13, "accuracy": 0.13, "wilson95": [ 0.07757167427240512, 0.20980351440076428 ], "head_correct": 81, "heads": 200, "head_accuracy": 0.405, "coverage": 1.0, "accuracy_on_successful": 0.13, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.07805473783678964, "brier": 0.494524471894728 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "correct": 68, "accuracy": 0.68, "wilson95": [ 0.5833738138954762, 0.7633085267500092 ], "head_correct": 68, "heads": 100, "head_accuracy": 0.68, "coverage": 1.0, "accuracy_on_successful": 0.68, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6280232039268183, "answered_single_label_heads": 100, "ece10": 0.1951895664150172, "brier": 0.40357124542077455 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "correct": 15, "accuracy": 0.15, "wilson95": [ 0.0930598528365663, 0.23283559590832364 ], "head_correct": 91, "heads": 200, "head_accuracy": 0.455, "coverage": 1.0, "accuracy_on_successful": 0.15, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.06163699502582122, "brier": 0.3129814599948459 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "head_correct": 220, "heads": 300, "head_accuracy": 0.7333333333333333, "coverage": 1.0, "accuracy_on_successful": 0.4, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.10876387428027996, "brier": 0.40365404116059606 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "head_correct": 60, "heads": 100, "head_accuracy": 0.6, "coverage": 1.0, "accuracy_on_successful": 0.6, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.580539138732416, "answered_single_label_heads": 100, "ece10": 0.21280714985920637, "brier": 0.5866908369214553 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "correct": 40, "accuracy": 0.4, "wilson95": [ 0.3094012864324589, 0.4979974132089382 ], "head_correct": 40, "heads": 100, "head_accuracy": 0.4, "coverage": 1.0, "accuracy_on_successful": 0.4, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.41572596649996024, "answered_single_label_heads": 100, "ece10": 0.10355152367087021, "brier": 0.7340497893310457 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "head_correct": 116, "heads": 300, "head_accuracy": 0.38666666666666666, "coverage": 1.0, "accuracy_on_successful": 0.04, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.12125852877616998, "brier": 0.6802449437085255 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 60, "accuracy": 0.6, "wilson95": [ 0.5020025867910618, 0.6905987135675411 ], "head_correct": 60, "heads": 100, "head_accuracy": 0.6, "coverage": 1.0, "accuracy_on_successful": 0.6, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5865362862589384, "answered_single_label_heads": 100, "ece10": 0.11965015482644088, "brier": 0.5603982062526418 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 335, "accuracy": 0.8375, "wilson95": [ 0.7981666097704109, 0.870412591553939 ], "head_correct": 335, "heads": 400, "head_accuracy": 0.8375, "coverage": 1.0, "accuracy_on_successful": 0.8375, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8224575610235569, "answered_single_label_heads": 400, "ece10": 0.04710430039239484, "brier": 0.2576419338029449 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 220, "accuracy": 0.55, "wilson95": [ 0.5010009985806122, 0.5980477719859582 ], "head_correct": 220, "heads": 400, "head_accuracy": 0.55, "coverage": 1.0, "accuracy_on_successful": 0.55, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.45769221283872447, "answered_single_label_heads": 400, "ece10": 0.060506538070317445, "brier": 0.5792296209440759 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 192, "accuracy": 0.48, "wilson95": [ 0.43146340667729527, 0.5289170850960765 ], "head_correct": 192, "heads": 400, "head_accuracy": 0.48, "coverage": 1.0, "accuracy_on_successful": 0.48, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.34043632673769664, "answered_single_label_heads": 400, "ece10": 0.2078332875390071, "brier": 0.5387235237127879 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 286, "accuracy": 0.715, "wilson95": [ 0.6688804833361408, 0.7570292301001116 ], "head_correct": 286, "heads": 400, "head_accuracy": 0.715, "coverage": 1.0, "accuracy_on_successful": 0.715, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6697949252693778, "answered_single_label_heads": 400, "ece10": 0.05543876591474991, "brier": 0.3904870271196406 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "correct": 350, "accuracy": 0.875, "wilson95": [ 0.8389809024860414, 0.9038848767632363 ], "head_correct": 350, "heads": 400, "head_accuracy": 0.875, "coverage": 1.0, "accuracy_on_successful": 0.875, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.775895298283358, "answered_single_label_heads": 400, "ece10": 0.10653332295555831, "brier": 0.20546767777123356 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "correct": 263, "accuracy": 0.6575, "wilson95": [ 0.6096946457889129, 0.7023089814957836 ], "head_correct": 263, "heads": 400, "head_accuracy": 0.6575, "coverage": 1.0, "accuracy_on_successful": 0.6575, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6277957767046343, "answered_single_label_heads": 400, "ece10": 0.08605053881177063, "brier": 0.4053591267508505 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "correct": 257, "accuracy": 0.8566666666666667, "wilson95": [ 0.8124994228503402, 0.8918152576761502 ], "head_correct": 257, "heads": 300, "head_accuracy": 0.8566666666666667, "coverage": 1.0, "accuracy_on_successful": 0.8566666666666667, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.07006341466871162, "brier": 0.21547097878814847 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "correct": 199, "accuracy": 0.6633333333333333, "wilson95": [ 0.6080919973085686, 0.7144446320914131 ], "head_correct": 199, "heads": 300, "head_accuracy": 0.6633333333333333, "coverage": 1.0, "accuracy_on_successful": 0.6633333333333333, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6621193705667738, "answered_single_label_heads": 300, "ece10": 0.06418397647225473, "brier": 0.43086377691953537 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "correct": 2149, "accuracy": 0.7289687924016283, "wilson95": [ 0.7126331699840927, 0.7447084653418594 ], "head_correct": 2149, "heads": 2948, "head_accuracy": 0.7289687924016283, "coverage": 1.0, "accuracy_on_successful": 0.7289687924016283, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.735920658505035, "answered_single_label_heads": 2948, "ece10": 0.030499996931015478, "brier": 0.38216662285406394 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "correct": 1614, "accuracy": 0.5474898236092266, "wilson95": [ 0.5294721556683573, 0.5653838872140624 ], "head_correct": 1614, "heads": 2948, "head_accuracy": 0.5474898236092266, "coverage": 1.0, "accuracy_on_successful": 0.5474898236092266, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5669663267562415, "answered_single_label_heads": 2948, "ece10": 0.03926874941231688, "brier": 0.5967794764432022 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "correct": 677, "accuracy": 0.577152600170503, "wilson95": [ 0.5486751333091117, 0.605126382301478 ], "head_correct": 677, "heads": 1173, "head_accuracy": 0.577152600170503, "coverage": 1.0, "accuracy_on_successful": 0.577152600170503, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.543275449679602, "answered_single_label_heads": 1173, "ece10": 0.06474422289344403, "brier": 0.5565252450099749 } } }, "alternatives_typed": { "expected": 400, "attempted": 400, "evaluation_expected": 400, "suites": { "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "correct": 15, "accuracy": 0.0375, "wilson95": [ 0.022855029814710867, 0.060943842444513396 ], "head_correct": 955, "heads": 2000, "head_accuracy": 0.4775, "coverage": 1.0, "accuracy_on_successful": 0.0375, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 2000, "ece10": 0.061353307895932754, "brier": 0.6168456580497913 } } } } }, "decision2-eos-08b": { "label": "Decision 2.0 Eos 0.8B", "source": "https://huggingface.co/vllm-sr/Decision-2.0-Eos-0.8B", "revision": "3594047d69f476f1d01cf84c593e213fc3a4dfe0", "public231": { "expected": 231, "attempted": 231, "complete": true, "correct": 163, "errors": 0, "tiers": { "jevbench_public/easy": { "n": 48, "correct": 48, "accuracy": 1.0, "strict_valid": 48, "valid": 48, "renormalized": 0 }, "jevbench_public/original": { "n": 72, "correct": 64, "accuracy": 0.8888888888888888, "strict_valid": 72, "valid": 72, "renormalized": 0 }, "jevbench_public/hard": { "n": 111, "correct": 51, "accuracy": 0.4594594594594595, "strict_valid": 111, "valid": 111, "renormalized": 0 } }, "accuracy": 0.7056277056277056 }, "shared_suites": { "jev_verified": { "expected": 300, "attempted": 300, "evaluation_expected": 300, "suites": { "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "correct": 73, "accuracy": 0.73, "wilson95": [ 0.6356788323205499, 0.8073041585042365 ], "head_correct": 73, "heads": 100, "head_accuracy": 0.73, "coverage": 1.0, "accuracy_on_successful": 0.73, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.680044275774826, "answered_single_label_heads": 100, "ece10": 0.10455175485564422, "brier": 0.39259620589630373 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "head_correct": 49, "heads": 100, "head_accuracy": 0.49, "coverage": 1.0, "accuracy_on_successful": 0.49, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.48719993723224436, "answered_single_label_heads": 100, "ece10": 0.2596564262057103, "brier": 0.6857648873726389 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "correct": 90, "accuracy": 0.9, "wilson95": [ 0.8256343384950865, 0.9447708629393249 ], "head_correct": 90, "heads": 100, "head_accuracy": 0.9, "coverage": 1.0, "accuracy_on_successful": 0.9, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8944444444444445, "answered_single_label_heads": 100, "ece10": 0.07063462327294892, "brier": 0.16211291910225345 } } }, "jev_fresh": { "expected": 1264, "attempted": 1264, "evaluation_expected": 848, "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "correct": 165, "accuracy": 0.825, "wilson95": [ 0.7663556885175099, 0.8713948493372666 ], "head_correct": 165, "heads": 200, "head_accuracy": 0.825, "coverage": 1.0, "accuracy_on_successful": 0.825, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.826392717083432, "answered_single_label_heads": 200, "ece10": 0.08007944954520693, "brier": 0.2984732049284324 }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "correct": 102, "accuracy": 0.51, "wilson95": [ 0.44118587678458643, 0.5784372166878682 ], "head_correct": 102, "heads": 200, "head_accuracy": 0.51, "coverage": 1.0, "accuracy_on_successful": 0.51, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.42346393140648875, "answered_single_label_heads": 200, "ece10": 0.1351521154929428, "brier": 0.6862455444425939 }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "correct": 139, "accuracy": 0.695, "wilson95": [ 0.6280144790202163, 0.7546358436926496 ], "head_correct": 139, "heads": 200, "head_accuracy": 0.695, "coverage": 1.0, "accuracy_on_successful": 0.695, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6922223063144882, "answered_single_label_heads": 200, "ece10": 0.17760066929130713, "brier": 0.448743850591924 }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "correct": 152, "accuracy": 0.76, "wilson95": [ 0.6962665902079583, 0.8139338400758631 ], "head_correct": 152, "heads": 200, "head_accuracy": 0.76, "coverage": 1.0, "accuracy_on_successful": 0.76, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7558742752517547, "answered_single_label_heads": 200, "ece10": 0.0525983687093, "brier": 0.3245369617838577 }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "correct": 6, "accuracy": 0.5, "wilson95": [ 0.2537815976337061, 0.7462184023662939 ], "head_correct": 6, "heads": 12, "head_accuracy": 0.5, "coverage": 1.0, "accuracy_on_successful": 0.5, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.4833333333333333, "answered_single_label_heads": 12, "ece10": 0.3521037869525475, "brier": 0.5864467682295508 }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "correct": 5, "accuracy": 0.4166666666666667, "wilson95": [ 0.19326031365875665, 0.6804886874504503 ], "head_correct": 5, "heads": 12, "head_accuracy": 0.4166666666666667, "coverage": 1.0, "accuracy_on_successful": 0.4166666666666667, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.37333333333333335, "answered_single_label_heads": 12, "ece10": 0.39604546993829554, "brier": 0.8714357142818826 }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "head_correct": 10, "heads": 12, "head_accuracy": 0.8333333333333334, "coverage": 1.0, "accuracy_on_successful": 0.8333333333333334, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.830952380952381, "answered_single_label_heads": 12, "ece10": 0.1387069444272858, "brier": 0.1279218060598574 }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "head_correct": 10, "heads": 12, "head_accuracy": 0.8333333333333334, "coverage": 1.0, "accuracy_on_successful": 0.8333333333333334, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8035714285714286, "answered_single_label_heads": 12, "ece10": 0.1525707492746187, "brier": 0.18602537149783735 } } }, "alternatives": { "expected": 12341, "attempted": 12341, "evaluation_expected": 12341, "suites": { "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "head_correct": 34, "heads": 40, "head_accuracy": 0.85, "coverage": 1.0, "accuracy_on_successful": 0.85, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8492063492063492, "answered_single_label_heads": 40, "ece10": 0.10482754674415927, "brier": 0.23288394723272293 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "correct": 33, "accuracy": 0.825, "wilson95": [ 0.6805000966821229, 0.9125458625396408 ], "head_correct": 33, "heads": 40, "head_accuracy": 0.825, "coverage": 1.0, "accuracy_on_successful": 0.825, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8273109243697478, "answered_single_label_heads": 40, "ece10": 0.1432725912530982, "brier": 0.2581527893575564 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "correct": 26, "accuracy": 0.65, "wilson95": [ 0.49505880837257693, 0.7786547112682372 ], "head_correct": 26, "heads": 40, "head_accuracy": 0.65, "coverage": 1.0, "accuracy_on_successful": 0.65, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6597108155002892, "answered_single_label_heads": 40, "ece10": 0.15712062003456062, "brier": 0.44277410161668296 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "correct": 27, "accuracy": 0.675, "wilson95": [ 0.5201774618987768, 0.799154977682173 ], "head_correct": 27, "heads": 40, "head_accuracy": 0.675, "coverage": 1.0, "accuracy_on_successful": 0.675, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6940409590409591, "answered_single_label_heads": 40, "ece10": 0.16015791175886462, "brier": 0.5213651964865484 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "head_correct": 28, "heads": 28, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 28, "ece10": 0.061463468819964076, "brier": 0.021228259895113892 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "head_correct": 28, "heads": 28, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 28, "ece10": 0.07576193456716397, "brier": 0.03853527132258527 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "correct": 25, "accuracy": 0.8928571428571429, "wilson95": [ 0.7280414845132785, 0.9628816331707 ], "head_correct": 25, "heads": 28, "head_accuracy": 0.8928571428571429, "coverage": 1.0, "accuracy_on_successful": 0.8928571428571429, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8897435897435899, "answered_single_label_heads": 28, "ece10": 0.07841131962770385, "brier": 0.10892584864610162 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "correct": 25, "accuracy": 0.8928571428571429, "wilson95": [ 0.7280414845132785, 0.9628816331707 ], "head_correct": 25, "heads": 28, "head_accuracy": 0.8928571428571429, "coverage": 1.0, "accuracy_on_successful": 0.8928571428571429, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8897435897435899, "answered_single_label_heads": 28, "ece10": 0.10151831356116436, "brier": 0.1465939354342535 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "correct": 82, "accuracy": 0.82, "wilson95": [ 0.7333264161369926, 0.8829977450105364 ], "head_correct": 82, "heads": 100, "head_accuracy": 0.82, "coverage": 1.0, "accuracy_on_successful": 0.82, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7997137451471817, "answered_single_label_heads": 100, "ece10": 0.05321251777598906, "brier": 0.267154745645119 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "head_correct": 49, "heads": 100, "head_accuracy": 0.49, "coverage": 1.0, "accuracy_on_successful": 0.49, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.4909615497850792, "answered_single_label_heads": 100, "ece10": 0.24124646916939363, "brier": 0.6586626550545032 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "correct": 84, "accuracy": 0.84, "wilson95": [ 0.7557973061072976, 0.8990471151119521 ], "head_correct": 84, "heads": 100, "head_accuracy": 0.84, "coverage": 1.0, "accuracy_on_successful": 0.84, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8393518518518518, "answered_single_label_heads": 100, "ece10": 0.05348144549442621, "brier": 0.198060777179602 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "correct": 65, "accuracy": 0.65, "wilson95": [ 0.5525444329688796, 0.7363575175690247 ], "head_correct": 65, "heads": 100, "head_accuracy": 0.65, "coverage": 1.0, "accuracy_on_successful": 0.65, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5792763553311696, "answered_single_label_heads": 100, "ece10": 0.1894682520004875, "brier": 0.5439533591104199 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "correct": 54, "accuracy": 0.54, "wilson95": [ 0.44264860323368227, 0.6343919169097589 ], "head_correct": 54, "heads": 100, "head_accuracy": 0.54, "coverage": 1.0, "accuracy_on_successful": 0.54, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5458400458078662, "answered_single_label_heads": 100, "ece10": 0.1484832876839354, "brier": 0.6458578177787117 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "head_correct": 138, "heads": 200, "head_accuracy": 0.69, "coverage": 1.0, "accuracy_on_successful": 0.44, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 200, "ece10": 0.10861779591777691, "brier": 0.4037580836277227 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "head_correct": 130, "heads": 200, "head_accuracy": 0.65, "coverage": 1.0, "accuracy_on_successful": 0.41, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 200, "ece10": 0.11847986508315161, "brier": 0.49875718545775966 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "head_correct": 76, "heads": 100, "head_accuracy": 0.76, "coverage": 1.0, "accuracy_on_successful": 0.76, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7018822494441689, "answered_single_label_heads": 100, "ece10": 0.07743750991002656, "brier": 0.33924274791738074 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "head_correct": 192, "heads": 400, "head_accuracy": 0.48, "coverage": 1.0, "accuracy_on_successful": 0.05, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 400, "ece10": 0.2828506170345381, "brier": 0.7264534965197559 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "correct": 56, "accuracy": 0.56, "wilson95": [ 0.4622810465167699, 0.6532797336983921 ], "head_correct": 56, "heads": 100, "head_accuracy": 0.56, "coverage": 1.0, "accuracy_on_successful": 0.56, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5512688703737296, "answered_single_label_heads": 100, "ece10": 0.15317063946273315, "brier": 0.5902145492008968 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "correct": 35, "accuracy": 0.35, "wilson95": [ 0.26364248243097527, 0.44745556703112027 ], "head_correct": 35, "heads": 100, "head_accuracy": 0.35, "coverage": 1.0, "accuracy_on_successful": 0.35, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.33846978091829355, "answered_single_label_heads": 100, "ece10": 0.1464477301898447, "brier": 0.776130161661398 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "correct": 4, "accuracy": 0.04, "wilson95": [ 0.01566330399154762, 0.0983707143588792 ], "head_correct": 65, "heads": 200, "head_accuracy": 0.325, "coverage": 1.0, "accuracy_on_successful": 0.04, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.20683249889156174, "brier": 0.6142779929117407 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "correct": 11, "accuracy": 0.11, "wilson95": [ 0.06254196357064568, 0.18631296503080313 ], "head_correct": 89, "heads": 200, "head_accuracy": 0.445, "coverage": 1.0, "accuracy_on_successful": 0.11, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.1365048534780268, "brier": 0.4019992178712932 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "correct": 69, "accuracy": 0.69, "wilson95": [ 0.5937394428051057, 0.7722030278762395 ], "head_correct": 69, "heads": 100, "head_accuracy": 0.69, "coverage": 1.0, "accuracy_on_successful": 0.69, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6388749651907547, "answered_single_label_heads": 100, "ece10": 0.18190625379493863, "brier": 0.4107982617110339 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "correct": 19, "accuracy": 0.19, "wilson95": [ 0.12514751509768735, 0.27778845379064376 ], "head_correct": 101, "heads": 200, "head_accuracy": 0.505, "coverage": 1.0, "accuracy_on_successful": 0.19, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.06662090387089349, "brier": 0.26720710588952845 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "head_correct": 216, "heads": 300, "head_accuracy": 0.72, "coverage": 1.0, "accuracy_on_successful": 0.41, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.13294084866999298, "brier": 0.392761921647207 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "head_correct": 58, "heads": 100, "head_accuracy": 0.58, "coverage": 1.0, "accuracy_on_successful": 0.58, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5402807859740634, "answered_single_label_heads": 100, "ece10": 0.0894188300828615, "brier": 0.5695480008927327 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "head_correct": 39, "heads": 100, "head_accuracy": 0.39, "coverage": 1.0, "accuracy_on_successful": 0.39, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.4147955707738317, "answered_single_label_heads": 100, "ece10": 0.23009233059165513, "brier": 0.827896101344405 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "head_correct": 131, "heads": 300, "head_accuracy": 0.43666666666666665, "coverage": 1.0, "accuracy_on_successful": 0.05, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.2135844783932972, "brier": 0.7573977279418347 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 67, "accuracy": 0.67, "wilson95": [ 0.5730534379509052, 0.7543687726587197 ], "head_correct": 67, "heads": 100, "head_accuracy": 0.67, "coverage": 1.0, "accuracy_on_successful": 0.67, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6447507378769238, "answered_single_label_heads": 100, "ece10": 0.09246153558655722, "brier": 0.4738765573005541 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 316, "accuracy": 0.79, "wilson95": [ 0.7474205944599498, 0.8270622748261584 ], "head_correct": 316, "heads": 400, "head_accuracy": 0.79, "coverage": 1.0, "accuracy_on_successful": 0.79, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7740969180674959, "answered_single_label_heads": 400, "ece10": 0.10576554114518641, "brier": 0.31872833618990426 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 211, "accuracy": 0.5275, "wilson95": [ 0.4785460270872868, 0.5759307967243268 ], "head_correct": 211, "heads": 400, "head_accuracy": 0.5275, "coverage": 1.0, "accuracy_on_successful": 0.5275, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.4649925707696623, "answered_single_label_heads": 400, "ece10": 0.0847491299323404, "brier": 0.629633437357299 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 300, "accuracy": 0.75, "wilson95": [ 0.7053228670693259, 0.7899209857635259 ], "head_correct": 300, "heads": 400, "head_accuracy": 0.75, "coverage": 1.0, "accuracy_on_successful": 0.75, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7406369955389562, "answered_single_label_heads": 400, "ece10": 0.11733714852799002, "brier": 0.3566888486461075 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 306, "accuracy": 0.765, "wilson95": [ 0.7210494588154521, 0.8039090251873708 ], "head_correct": 306, "heads": 400, "head_accuracy": 0.765, "coverage": 1.0, "accuracy_on_successful": 0.765, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7599530120789602, "answered_single_label_heads": 400, "ece10": 0.06989510769675708, "brier": 0.3455845205119977 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "correct": 370, "accuracy": 0.925, "wilson95": [ 0.8949522937386957, 0.9469622560771523 ], "head_correct": 370, "heads": 400, "head_accuracy": 0.925, "coverage": 1.0, "accuracy_on_successful": 0.925, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8828125, "answered_single_label_heads": 400, "ece10": 0.05792060191897319, "brier": 0.12325601266760891 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "correct": 280, "accuracy": 0.7, "wilson95": [ 0.6533627542224784, 0.7428323280438028 ], "head_correct": 280, "heads": 400, "head_accuracy": 0.7, "coverage": 1.0, "accuracy_on_successful": 0.7, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6808425756004148, "answered_single_label_heads": 400, "ece10": 0.06741079435616933, "brier": 0.41707686885730266 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "correct": 254, "accuracy": 0.8466666666666667, "wilson95": [ 0.8015339429418983, 0.8830335970090831 ], "head_correct": 254, "heads": 300, "head_accuracy": 0.8466666666666667, "coverage": 1.0, "accuracy_on_successful": 0.8466666666666667, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.03855331808856283, "brier": 0.2118956389779299 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "correct": 230, "accuracy": 0.7666666666666667, "wilson95": [ 0.715618653578636, 0.8109717617682728 ], "head_correct": 230, "heads": 300, "head_accuracy": 0.7666666666666667, "coverage": 1.0, "accuracy_on_successful": 0.7666666666666667, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.724965719394862, "answered_single_label_heads": 300, "ece10": 0.056644465304332536, "brier": 0.35736411747108887 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "correct": 1989, "accuracy": 0.6746947082767978, "wilson95": [ 0.6575652873232078, 0.6913694418514074 ], "head_correct": 1989, "heads": 2948, "head_accuracy": 0.6746947082767978, "coverage": 1.0, "accuracy_on_successful": 0.6746947082767978, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6735496983713866, "answered_single_label_heads": 2948, "ece10": 0.14691230397846916, "brier": 0.4901519826634882 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "correct": 1551, "accuracy": 0.5261194029850746, "wilson95": [ 0.508072705278896, 0.5440981183064348 ], "head_correct": 1551, "heads": 2948, "head_accuracy": 0.5261194029850746, "coverage": 1.0, "accuracy_on_successful": 0.5261194029850746, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5346075226507654, "answered_single_label_heads": 2948, "ece10": 0.19417659918332386, "brier": 0.6700835391326051 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "correct": 626, "accuracy": 0.5336743393009378, "wilson95": [ 0.5050624307833703, 0.5620664075218041 ], "head_correct": 626, "heads": 1173, "head_accuracy": 0.5336743393009378, "coverage": 1.0, "accuracy_on_successful": 0.5336743393009378, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.508786544146597, "answered_single_label_heads": 1173, "ece10": 0.16360546283881439, "brier": 0.6029639859874418 } } }, "alternatives_typed": { "expected": 400, "attempted": 400, "evaluation_expected": 400, "suites": { "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "correct": 8, "accuracy": 0.02, "wilson95": [ 0.010168386791664645, 0.03896341576926 ], "head_correct": 922, "heads": 2000, "head_accuracy": 0.461, "coverage": 1.0, "accuracy_on_successful": 0.02, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 2000, "ece10": 0.2075431872315776, "brier": 0.710161608511819 } } } } }, "decision2-sol-2b": { "label": "Decision 2.0 Sol 2B", "source": "https://huggingface.co/vllm-sr/Decision-2.0-Sol-2B", "revision": "23cbe9f96dde4a7e1e6a238d576128bf73c5b09e", "public231": { "expected": 231, "attempted": 231, "complete": true, "correct": 166, "errors": 0, "tiers": { "jevbench_public/easy": { "n": 48, "correct": 48, "accuracy": 1.0, "strict_valid": 48, "valid": 48, "renormalized": 0 }, "jevbench_public/original": { "n": 72, "correct": 67, "accuracy": 0.9305555555555556, "strict_valid": 72, "valid": 72, "renormalized": 0 }, "jevbench_public/hard": { "n": 111, "correct": 51, "accuracy": 0.4594594594594595, "strict_valid": 111, "valid": 111, "renormalized": 0 } }, "accuracy": 0.7186147186147186 }, "shared_suites": { "jev_verified": { "expected": 300, "attempted": 300, "evaluation_expected": 300, "suites": { "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "head_correct": 80, "heads": 100, "head_accuracy": 0.8, "coverage": 1.0, "accuracy_on_successful": 0.8, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7683881335553089, "answered_single_label_heads": 100, "ece10": 0.07808215492913945, "brier": 0.3094013777239084 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "correct": 52, "accuracy": 0.52, "wilson95": [ 0.4231657776522397, 0.615354482419481 ], "head_correct": 52, "heads": 100, "head_accuracy": 0.52, "coverage": 1.0, "accuracy_on_successful": 0.52, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.4995189534035706, "answered_single_label_heads": 100, "ece10": 0.15807219682167148, "brier": 0.62230564883843 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "correct": 89, "accuracy": 0.89, "wilson95": [ 0.8136870349691969, 0.9374580364293543 ], "head_correct": 89, "heads": 100, "head_accuracy": 0.89, "coverage": 1.0, "accuracy_on_successful": 0.89, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8837962962962963, "answered_single_label_heads": 100, "ece10": 0.1219715402750424, "brier": 0.19393854356598175 } } }, "jev_fresh": { "expected": 1264, "attempted": 1264, "evaluation_expected": 848, "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "correct": 167, "accuracy": 0.835, "wilson95": [ 0.7773421846094045, 0.8800314467178268 ], "head_correct": 167, "heads": 200, "head_accuracy": 0.835, "coverage": 1.0, "accuracy_on_successful": 0.835, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8358803069477516, "answered_single_label_heads": 200, "ece10": 0.039382858870365405, "brier": 0.23195408765393105 }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "correct": 58, "accuracy": 0.29, "wilson95": [ 0.23154025607067882, 0.3563747810077731 ], "head_correct": 58, "heads": 200, "head_accuracy": 0.29, "coverage": 1.0, "accuracy_on_successful": 0.29, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.09528666751668154, "answered_single_label_heads": 200, "ece10": 0.07738324240289889, "brier": 0.8155000339764635 }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "correct": 149, "accuracy": 0.745, "wilson95": [ 0.6803707417954529, 0.8003950482796865 ], "head_correct": 149, "heads": 200, "head_accuracy": 0.745, "coverage": 1.0, "accuracy_on_successful": 0.745, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7439180537772088, "answered_single_label_heads": 200, "ece10": 0.06877994921058281, "brier": 0.3403342681089129 }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "correct": 172, "accuracy": 0.86, "wilson95": [ 0.8051011913429243, 0.9013301736654437 ], "head_correct": 172, "heads": 200, "head_accuracy": 0.86, "coverage": 1.0, "accuracy_on_successful": 0.86, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8514431239388794, "answered_single_label_heads": 200, "ece10": 0.05861484046555323, "brier": 0.23538100457286404 }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "correct": 7, "accuracy": 0.5833333333333334, "wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "head_correct": 7, "heads": 12, "head_accuracy": 0.5833333333333334, "coverage": 1.0, "accuracy_on_successful": 0.5833333333333334, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5555555555555556, "answered_single_label_heads": 12, "ece10": 0.23117896642679361, "brier": 0.42842671963849815 }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "correct": 7, "accuracy": 0.5833333333333334, "wilson95": [ 0.31951131254954973, 0.8067396863412435 ], "head_correct": 7, "heads": 12, "head_accuracy": 0.5833333333333334, "coverage": 1.0, "accuracy_on_successful": 0.5833333333333334, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5504761904761905, "answered_single_label_heads": 12, "ece10": 0.12113809839010516, "brier": 0.5830241232759327 }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "head_correct": 12, "heads": 12, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 12, "ece10": 0.08244879735252686, "brier": 0.02217850312387622 }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "head_correct": 12, "heads": 12, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 12, "ece10": 0.13675118534536745, "brier": 0.05248382037510425 } } }, "alternatives": { "expected": 12341, "attempted": 12341, "evaluation_expected": 12341, "suites": { "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "head_correct": 35, "heads": 40, "head_accuracy": 0.875, "coverage": 1.0, "accuracy_on_successful": 0.875, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8804636591478696, "answered_single_label_heads": 40, "ece10": 0.19371464405577096, "brier": 0.2449728866932142 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "correct": 31, "accuracy": 0.775, "wilson95": [ 0.6249690325768998, 0.8768390867645927 ], "head_correct": 31, "heads": 40, "head_accuracy": 0.775, "coverage": 1.0, "accuracy_on_successful": 0.775, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7725763125763127, "answered_single_label_heads": 40, "ece10": 0.16349289143399845, "brier": 0.2856541036449992 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "correct": 30, "accuracy": 0.75, "wilson95": [ 0.5980603857923197, 0.858128813609037 ], "head_correct": 30, "heads": 40, "head_accuracy": 0.75, "coverage": 1.0, "accuracy_on_successful": 0.75, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7588461538461538, "answered_single_label_heads": 40, "ece10": 0.13051176375969467, "brier": 0.4038104223237731 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "correct": 28, "accuracy": 0.7, "wilson95": [ 0.5456998118185507, 0.8192515477025347 ], "head_correct": 28, "heads": 40, "head_accuracy": 0.7, "coverage": 1.0, "accuracy_on_successful": 0.7, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7047619047619047, "answered_single_label_heads": 40, "ece10": 0.1413429391457714, "brier": 0.40460690387556825 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "head_correct": 28, "heads": 28, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 28, "ece10": 0.07568117735070633, "brier": 0.021222758396229637 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "head_correct": 27, "heads": 28, "head_accuracy": 0.9642857142857143, "coverage": 1.0, "accuracy_on_successful": 0.9642857142857143, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9641025641025642, "answered_single_label_heads": 28, "ece10": 0.11943105964889364, "brier": 0.05283006751609493 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "head_correct": 28, "heads": 28, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 28, "ece10": 0.08926798044883773, "brier": 0.023868708079755625 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "head_correct": 27, "heads": 28, "head_accuracy": 0.9642857142857143, "coverage": 1.0, "accuracy_on_successful": 0.9642857142857143, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9641025641025642, "answered_single_label_heads": 28, "ece10": 0.12158132694568302, "brier": 0.08984177103004642 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "head_correct": 80, "heads": 100, "head_accuracy": 0.8, "coverage": 1.0, "accuracy_on_successful": 0.8, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7854016550825061, "answered_single_label_heads": 100, "ece10": 0.06880886073700877, "brier": 0.2487869154143886 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "head_correct": 49, "heads": 100, "head_accuracy": 0.49, "coverage": 1.0, "accuracy_on_successful": 0.49, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.4889382783404523, "answered_single_label_heads": 100, "ece10": 0.207736097870836, "brier": 0.6431851926066946 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "correct": 90, "accuracy": 0.9, "wilson95": [ 0.8256343384950865, 0.9447708629393249 ], "head_correct": 90, "heads": 100, "head_accuracy": 0.9, "coverage": 1.0, "accuracy_on_successful": 0.9, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8912037037037036, "answered_single_label_heads": 100, "ece10": 0.12821954524762152, "brier": 0.19889295141911487 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "correct": 75, "accuracy": 0.75, "wilson95": [ 0.656955364519384, 0.8245478863771232 ], "head_correct": 75, "heads": 100, "head_accuracy": 0.75, "coverage": 1.0, "accuracy_on_successful": 0.75, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7333333333333334, "answered_single_label_heads": 100, "ece10": 0.059437624941391695, "brier": 0.3660810709197845 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "correct": 55, "accuracy": 0.55, "wilson95": [ 0.45244602997442135, 0.6438546202048803 ], "head_correct": 55, "heads": 100, "head_accuracy": 0.55, "coverage": 1.0, "accuracy_on_successful": 0.55, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5357259215635175, "answered_single_label_heads": 100, "ece10": 0.0922466182484411, "brier": 0.6015201792387319 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 50, "accuracy": 0.5, "wilson95": [ 0.4038315303659956, 0.5961684696340044 ], "head_correct": 144, "heads": 200, "head_accuracy": 0.72, "coverage": 1.0, "accuracy_on_successful": 0.5, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 200, "ece10": 0.08153269226712541, "brier": 0.3842651821117649 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 38, "accuracy": 0.38, "wilson95": [ 0.29097599252478734, 0.47790244704488916 ], "head_correct": 125, "heads": 200, "head_accuracy": 0.625, "coverage": 1.0, "accuracy_on_successful": 0.38, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 200, "ece10": 0.07340707112581586, "brier": 0.47832212599657997 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "head_correct": 80, "heads": 100, "head_accuracy": 0.8, "coverage": 1.0, "accuracy_on_successful": 0.8, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7491205500133119, "answered_single_label_heads": 100, "ece10": 0.08729988193788625, "brier": 0.340338436666961 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "correct": 7, "accuracy": 0.07, "wilson95": [ 0.03431926106727268, 0.137495147390735 ], "head_correct": 206, "heads": 400, "head_accuracy": 0.515, "coverage": 1.0, "accuracy_on_successful": 0.07, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 400, "ece10": 0.22052586147484474, "brier": 0.7049394635209565 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "head_correct": 58, "heads": 100, "head_accuracy": 0.58, "coverage": 1.0, "accuracy_on_successful": 0.58, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5622239619311903, "answered_single_label_heads": 100, "ece10": 0.13386536972081378, "brier": 0.5509757939815427 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "head_correct": 41, "heads": 100, "head_accuracy": 0.41, "coverage": 1.0, "accuracy_on_successful": 0.41, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.38605548025661884, "answered_single_label_heads": 100, "ece10": 0.08078339929395337, "brier": 0.7357031221127025 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "correct": 3, "accuracy": 0.03, "wilson95": [ 0.010254524024038925, 0.0845193642905276 ], "head_correct": 68, "heads": 200, "head_accuracy": 0.34, "coverage": 1.0, "accuracy_on_successful": 0.03, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.08963781350552533, "brier": 0.5069165762092337 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "correct": 19, "accuracy": 0.19, "wilson95": [ 0.12514751509768735, 0.27778845379064376 ], "head_correct": 102, "heads": 200, "head_accuracy": 0.51, "coverage": 1.0, "accuracy_on_successful": 0.19, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.09250068984968213, "brier": 0.36318823495473573 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "correct": 76, "accuracy": 0.76, "wilson95": [ 0.6676766365018553, 0.8330867444305122 ], "head_correct": 76, "heads": 100, "head_accuracy": 0.76, "coverage": 1.0, "accuracy_on_successful": 0.76, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7498889066638226, "answered_single_label_heads": 100, "ece10": 0.10542693973034115, "brier": 0.35173023530399233 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "correct": 18, "accuracy": 0.18, "wilson95": [ 0.1170022549894635, 0.2666735838630073 ], "head_correct": 100, "heads": 200, "head_accuracy": 0.5, "coverage": 1.0, "accuracy_on_successful": 0.18, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.09269865931462724, "brier": 0.27632859578900426 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "head_correct": 229, "heads": 300, "head_accuracy": 0.7633333333333333, "coverage": 1.0, "accuracy_on_successful": 0.44, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.025166218706536132, "brier": 0.3509157403784562 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "correct": 55, "accuracy": 0.55, "wilson95": [ 0.45244602997442135, 0.6438546202048803 ], "head_correct": 55, "heads": 100, "head_accuracy": 0.55, "coverage": 1.0, "accuracy_on_successful": 0.55, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.54027305560919, "answered_single_label_heads": 100, "ece10": 0.1501719737746777, "brier": 0.5918594055699193 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "correct": 49, "accuracy": 0.49, "wilson95": [ 0.3942199893044113, 0.5865198806597283 ], "head_correct": 49, "heads": 100, "head_accuracy": 0.49, "coverage": 1.0, "accuracy_on_successful": 0.49, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.4917492229992229, "answered_single_label_heads": 100, "ece10": 0.07835742492090617, "brier": 0.7128276173154156 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "correct": 11, "accuracy": 0.11, "wilson95": [ 0.06254196357064568, 0.18631296503080313 ], "head_correct": 147, "heads": 300, "head_accuracy": 0.49, "coverage": 1.0, "accuracy_on_successful": 0.11, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.058607769691736145, "brier": 0.6450734140324356 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 66, "accuracy": 0.66, "wilson95": [ 0.5627772885472462, 0.7453847920265184 ], "head_correct": 66, "heads": 100, "head_accuracy": 0.66, "coverage": 1.0, "accuracy_on_successful": 0.66, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6390741913458632, "answered_single_label_heads": 100, "ece10": 0.12051706228728479, "brier": 0.5321099923317381 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 328, "accuracy": 0.82, "wilson95": [ 0.7793624469518832, 0.8545496846741669 ], "head_correct": 328, "heads": 400, "head_accuracy": 0.82, "coverage": 1.0, "accuracy_on_successful": 0.82, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8029231453826189, "answered_single_label_heads": 400, "ece10": 0.07587187458735661, "brier": 0.2684407259700472 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 126, "accuracy": 0.315, "wilson95": [ 0.2714209156648816, 0.3620986332388081 ], "head_correct": 126, "heads": 400, "head_accuracy": 0.315, "coverage": 1.0, "accuracy_on_successful": 0.315, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.11412583916058205, "answered_single_label_heads": 400, "ece10": 0.10085037181619065, "brier": 0.8115939551891656 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 320, "accuracy": 0.8, "wilson95": [ 0.758029683133299, 0.8362629402661231 ], "head_correct": 320, "heads": 400, "head_accuracy": 0.8, "coverage": 1.0, "accuracy_on_successful": 0.8, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7993931643220743, "answered_single_label_heads": 400, "ece10": 0.0465617772911906, "brier": 0.2738771950354416 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 341, "accuracy": 0.8525, "wilson95": [ 0.8143999058527144, 0.8838939266416067 ], "head_correct": 341, "heads": 400, "head_accuracy": 0.8525, "coverage": 1.0, "accuracy_on_successful": 0.8525, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8471255060728745, "answered_single_label_heads": 400, "ece10": 0.057849737551275746, "brier": 0.22233104026828937 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "correct": 367, "accuracy": 0.9175, "wilson95": [ 0.8864031104605777, 0.9406541237702847 ], "head_correct": 367, "heads": 400, "head_accuracy": 0.9175, "coverage": 1.0, "accuracy_on_successful": 0.9175, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8856935027147793, "answered_single_label_heads": 400, "ece10": 0.06758592205181282, "brier": 0.13321024752441288 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "correct": 316, "accuracy": 0.79, "wilson95": [ 0.7474205944599498, 0.8270622748261584 ], "head_correct": 316, "heads": 400, "head_accuracy": 0.79, "coverage": 1.0, "accuracy_on_successful": 0.79, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7892412685668406, "answered_single_label_heads": 400, "ece10": 0.055856382336404276, "brier": 0.3258298909494575 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "correct": 254, "accuracy": 0.8466666666666667, "wilson95": [ 0.8015339429418983, 0.8830335970090831 ], "head_correct": 254, "heads": 300, "head_accuracy": 0.8466666666666667, "coverage": 1.0, "accuracy_on_successful": 0.8466666666666667, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.05422101177321819, "brier": 0.19468136682980508 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "correct": 217, "accuracy": 0.7233333333333334, "wilson95": [ 0.6701300560341223, 0.7708894168189138 ], "head_correct": 217, "heads": 300, "head_accuracy": 0.7233333333333334, "coverage": 1.0, "accuracy_on_successful": 0.7233333333333334, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6989892901100611, "answered_single_label_heads": 300, "ece10": 0.06595443036733628, "brier": 0.3922997052768056 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "correct": 2165, "accuracy": 0.7343962008141113, "wilson95": [ 0.7181557430936916, 0.7500265828473942 ], "head_correct": 2165, "heads": 2948, "head_accuracy": 0.7343962008141113, "coverage": 1.0, "accuracy_on_successful": 0.7343962008141113, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7246696078366311, "answered_single_label_heads": 2948, "ece10": 0.04883348534706847, "brier": 0.3814263240443554 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "correct": 1898, "accuracy": 0.6438263229308006, "wilson95": [ 0.6263631846040778, 0.6609151166969647 ], "head_correct": 1898, "heads": 2948, "head_accuracy": 0.6438263229308006, "coverage": 1.0, "accuracy_on_successful": 0.6438263229308006, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6461877179332203, "answered_single_label_heads": 2948, "ece10": 0.06535133300598188, "brier": 0.4862412561185921 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "correct": 671, "accuracy": 0.5720375106564365, "wilson95": [ 0.5435327924347861, 0.6000719376104604 ], "head_correct": 671, "heads": 1173, "head_accuracy": 0.5720375106564365, "coverage": 1.0, "accuracy_on_successful": 0.5720375106564365, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5618515858531928, "answered_single_label_heads": 1173, "ece10": 0.05439335389477179, "brier": 0.5329661644692806 } } }, "alternatives_typed": { "expected": 400, "attempted": 400, "evaluation_expected": 400, "suites": { "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "correct": 25, "accuracy": 0.0625, "wilson95": [ 0.04268914141220244, 0.09063411613030699 ], "head_correct": 1102, "heads": 2000, "head_accuracy": 0.551, "coverage": 1.0, "accuracy_on_successful": 0.0625, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 2000, "ece10": 0.05980603739468885, "brier": 0.555920117850975 } } } } }, "decision2-nox-4b": { "label": "Decision 2.0 Nox 4B", "source": "https://huggingface.co/vllm-sr/Decision-2.0-Nox-4B", "revision": "25e8f67d1b486c647222df3aac640d2d5d736bbe", "public231": { "expected": 231, "attempted": 231, "complete": true, "correct": 185, "errors": 0, "tiers": { "jevbench_public/easy": { "n": 48, "correct": 48, "accuracy": 1.0, "strict_valid": 48, "valid": 48, "renormalized": 0 }, "jevbench_public/original": { "n": 72, "correct": 68, "accuracy": 0.9444444444444444, "strict_valid": 72, "valid": 72, "renormalized": 0 }, "jevbench_public/hard": { "n": 111, "correct": 69, "accuracy": 0.6216216216216216, "strict_valid": 111, "valid": 111, "renormalized": 0 } }, "accuracy": 0.8008658008658008 }, "shared_suites": { "jev_verified": { "expected": 300, "attempted": 300, "evaluation_expected": 300, "suites": { "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "correct": 85, "accuracy": 0.85, "wilson95": [ 0.7671644040916763, 0.9069401471634337 ], "head_correct": 85, "heads": 100, "head_accuracy": 0.85, "coverage": 1.0, "accuracy_on_successful": 0.85, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.829952700683251, "answered_single_label_heads": 100, "ece10": 0.08115424039441346, "brier": 0.21375037475498065 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "correct": 51, "accuracy": 0.51, "wilson95": [ 0.4134801193402717, 0.6057800106955886 ], "head_correct": 51, "heads": 100, "head_accuracy": 0.51, "coverage": 1.0, "accuracy_on_successful": 0.51, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.512612734192676, "answered_single_label_heads": 100, "ece10": 0.13195013443187742, "brier": 0.6377739569814431 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "correct": 86, "accuracy": 0.86, "wilson95": [ 0.7786281178903616, 0.9147365634006086 ], "head_correct": 86, "heads": 100, "head_accuracy": 0.86, "coverage": 1.0, "accuracy_on_successful": 0.86, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8574074074074073, "answered_single_label_heads": 100, "ece10": 0.11018888794998132, "brier": 0.23113922068027315 } } }, "jev_fresh": { "expected": 1264, "attempted": 1264, "evaluation_expected": 848, "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "correct": 174, "accuracy": 0.87, "wilson95": [ 0.8163364812775159, 0.9097179772033067 ], "head_correct": 174, "heads": 200, "head_accuracy": 0.87, "coverage": 1.0, "accuracy_on_successful": 0.87, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8726788168578128, "answered_single_label_heads": 200, "ece10": 0.05494446091773509, "brier": 0.19891208857638365 }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "correct": 118, "accuracy": 0.59, "wilson95": [ 0.5207645903369208, 0.6558432509151711 ], "head_correct": 118, "heads": 200, "head_accuracy": 0.59, "coverage": 1.0, "accuracy_on_successful": 0.59, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.461245944971765, "answered_single_label_heads": 200, "ece10": 0.12635538256318152, "brier": 0.61429039464794 }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "correct": 186, "accuracy": 0.93, "wilson95": [ 0.8859451911438936, 0.9578478281716571 ], "head_correct": 186, "heads": 200, "head_accuracy": 0.93, "coverage": 1.0, "accuracy_on_successful": 0.93, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9294283697953423, "answered_single_label_heads": 200, "ece10": 0.051246475070312644, "brier": 0.1209913187821238 }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "correct": 185, "accuracy": 0.925, "wilson95": [ 0.8799563897637055, 0.954025082815618 ], "head_correct": 185, "heads": 200, "head_accuracy": 0.925, "coverage": 1.0, "accuracy_on_successful": 0.925, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9206160196872271, "answered_single_label_heads": 200, "ece10": 0.060558283396076835, "brier": 0.11696930074108831 }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "head_correct": 10, "heads": 12, "head_accuracy": 0.8333333333333334, "coverage": 1.0, "accuracy_on_successful": 0.8333333333333334, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8533333333333333, "answered_single_label_heads": 12, "ece10": 0.13423910085356627, "brier": 0.1973404436470384 }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "correct": 10, "accuracy": 0.8333333333333334, "wilson95": [ 0.5519691377470266, 0.9530348578161463 ], "head_correct": 10, "heads": 12, "head_accuracy": 0.8333333333333334, "coverage": 1.0, "accuracy_on_successful": 0.8333333333333334, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8533333333333333, "answered_single_label_heads": 12, "ece10": 0.21504290432537748, "brier": 0.30312594381380104 }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "head_correct": 11, "heads": 12, "head_accuracy": 0.9166666666666666, "coverage": 1.0, "accuracy_on_successful": 0.9166666666666666, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9142857142857143, "answered_single_label_heads": 12, "ece10": 0.09784653392831888, "brier": 0.065169349697962 }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "head_correct": 12, "heads": 12, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 12, "ece10": 0.06361458369723726, "brier": 0.01778003201564767 } } }, "alternatives": { "expected": 12341, "attempted": 12341, "evaluation_expected": 12341, "suites": { "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "head_correct": 38, "heads": 40, "head_accuracy": 0.95, "coverage": 1.0, "accuracy_on_successful": 0.95, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9498039215686275, "answered_single_label_heads": 40, "ece10": 0.09826164089761917, "brier": 0.10381527113289994 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "head_correct": 38, "heads": 40, "head_accuracy": 0.95, "coverage": 1.0, "accuracy_on_successful": 0.95, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9498039215686275, "answered_single_label_heads": 40, "ece10": 0.11505517163475147, "brier": 0.13641782933683821 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "correct": 35, "accuracy": 0.875, "wilson95": [ 0.738878801611489, 0.945404997490546 ], "head_correct": 35, "heads": 40, "head_accuracy": 0.875, "coverage": 1.0, "accuracy_on_successful": 0.875, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8625816993464053, "answered_single_label_heads": 40, "ece10": 0.11931748225382628, "brier": 0.16712662820186627 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "correct": 34, "accuracy": 0.85, "wilson95": [ 0.709276756335103, 0.9293881228267964 ], "head_correct": 34, "heads": 40, "head_accuracy": 0.85, "coverage": 1.0, "accuracy_on_successful": 0.85, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8415686274509804, "answered_single_label_heads": 40, "ece10": 0.08488533530926969, "brier": 0.2244684670349657 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "head_correct": 28, "heads": 28, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 28, "ece10": 0.042052975183637015, "brier": 0.008709071702229102 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "head_correct": 27, "heads": 28, "head_accuracy": 0.9642857142857143, "coverage": 1.0, "accuracy_on_successful": 0.9642857142857143, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9641025641025642, "answered_single_label_heads": 28, "ece10": 0.055568611295082525, "brier": 0.027459085021526775 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "head_correct": 28, "heads": 28, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 28, "ece10": 0.06513701731431297, "brier": 0.016751869651559066 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "head_correct": 28, "heads": 28, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 28, "ece10": 0.06980432875210964, "brier": 0.01784443553567067 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "head_correct": 88, "heads": 100, "head_accuracy": 0.88, "coverage": 1.0, "accuracy_on_successful": 0.88, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8683996929589101, "answered_single_label_heads": 100, "ece10": 0.06748331016314214, "brier": 0.16251029809196266 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "correct": 53, "accuracy": 0.53, "wilson95": [ 0.4328885697009936, 0.6248918204065872 ], "head_correct": 53, "heads": 100, "head_accuracy": 0.53, "coverage": 1.0, "accuracy_on_successful": 0.53, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5368247358680628, "answered_single_label_heads": 100, "ece10": 0.09724809071617313, "brier": 0.616309006941107 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "head_correct": 87, "heads": 100, "head_accuracy": 0.87, "coverage": 1.0, "accuracy_on_successful": 0.87, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8537698412698412, "answered_single_label_heads": 100, "ece10": 0.07383694983749373, "brier": 0.21507851841288608 }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "correct": 81, "accuracy": 0.81, "wilson95": [ 0.7222115462093562, 0.8748524849023126 ], "head_correct": 81, "heads": 100, "head_accuracy": 0.81, "coverage": 1.0, "accuracy_on_successful": 0.81, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.801234438748823, "answered_single_label_heads": 100, "ece10": 0.12652606501322633, "brier": 0.3028627977667988 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "correct": 59, "accuracy": 0.59, "wilson95": [ 0.4920143005341078, 0.6813268697886348 ], "head_correct": 59, "heads": 100, "head_accuracy": 0.59, "coverage": 1.0, "accuracy_on_successful": 0.59, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6008085664335664, "answered_single_label_heads": 100, "ece10": 0.07536757380431143, "brier": 0.579849569947826 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 47, "accuracy": 0.47, "wilson95": [ 0.3751081795934127, 0.5671114302990063 ], "head_correct": 143, "heads": 200, "head_accuracy": 0.715, "coverage": 1.0, "accuracy_on_successful": 0.47, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 200, "ece10": 0.12210132174466944, "brier": 0.37284597156968374 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "head_correct": 134, "heads": 200, "head_accuracy": 0.67, "coverage": 1.0, "accuracy_on_successful": 0.48, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 200, "ece10": 0.03914290542988093, "brier": 0.4535490027596118 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "correct": 80, "accuracy": 0.8, "wilson95": [ 0.7111708344068411, 0.8666330666689676 ], "head_correct": 80, "heads": 100, "head_accuracy": 0.8, "coverage": 1.0, "accuracy_on_successful": 0.8, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7439629270511624, "answered_single_label_heads": 100, "ece10": 0.05683705675187247, "brier": 0.3135056859617968 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "head_correct": 220, "heads": 400, "head_accuracy": 0.55, "coverage": 1.0, "accuracy_on_successful": 0.08, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 400, "ece10": 0.21416612045648492, "brier": 0.6775720351881234 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "correct": 67, "accuracy": 0.67, "wilson95": [ 0.5730534379509052, 0.7543687726587197 ], "head_correct": 67, "heads": 100, "head_accuracy": 0.67, "coverage": 1.0, "accuracy_on_successful": 0.67, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6718214632916477, "answered_single_label_heads": 100, "ece10": 0.09265499714688744, "brier": 0.48893706894393324 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "correct": 43, "accuracy": 0.43, "wilson95": [ 0.33733298524110106, 0.5278461045078768 ], "head_correct": 43, "heads": 100, "head_accuracy": 0.43, "coverage": 1.0, "accuracy_on_successful": 0.43, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.40832268464210386, "answered_single_label_heads": 100, "ece10": 0.04794223136148084, "brier": 0.7049597117538426 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "correct": 13, "accuracy": 0.13, "wilson95": [ 0.07757167427240512, 0.20980351440076428 ], "head_correct": 80, "heads": 200, "head_accuracy": 0.4, "coverage": 1.0, "accuracy_on_successful": 0.13, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.1633931716106146, "brier": 0.485163958712508 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "correct": 42, "accuracy": 0.42, "wilson95": [ 0.3279838267435473, 0.5179351329695703 ], "head_correct": 132, "heads": 200, "head_accuracy": 0.66, "coverage": 1.0, "accuracy_on_successful": 0.42, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.16755774786856048, "brier": 0.31948512778067795 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "correct": 87, "accuracy": 0.87, "wilson95": [ 0.7901964855992357, 0.9224283257275948 ], "head_correct": 87, "heads": 100, "head_accuracy": 0.87, "coverage": 1.0, "accuracy_on_successful": 0.87, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8699055330634279, "answered_single_label_heads": 100, "ece10": 0.16611165750960535, "brier": 0.24327089555536877 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "correct": 19, "accuracy": 0.19, "wilson95": [ 0.12514751509768735, 0.27778845379064376 ], "head_correct": 109, "heads": 200, "head_accuracy": 0.545, "coverage": 1.0, "accuracy_on_successful": 0.19, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.07977147658598066, "brier": 0.22893162553061486 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "head_correct": 234, "heads": 300, "head_accuracy": 0.78, "coverage": 1.0, "accuracy_on_successful": 0.48, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.018250654216557652, "brier": 0.30368235275275723 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "correct": 69, "accuracy": 0.69, "wilson95": [ 0.5937394428051057, 0.7722030278762395 ], "head_correct": 69, "heads": 100, "head_accuracy": 0.69, "coverage": 1.0, "accuracy_on_successful": 0.69, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6489191713981631, "answered_single_label_heads": 100, "ece10": 0.12274612093309654, "brier": 0.43580125982568807 }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "correct": 44, "accuracy": 0.44, "wilson95": [ 0.34672026630160796, 0.5377189534832302 ], "head_correct": 44, "heads": 100, "head_accuracy": 0.44, "coverage": 1.0, "accuracy_on_successful": 0.44, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.4404621958501269, "answered_single_label_heads": 100, "ece10": 0.13523129686613852, "brier": 0.7179921059037478 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "correct": 12, "accuracy": 0.12, "wilson95": [ 0.06999406437019488, 0.19812099426711421 ], "head_correct": 154, "heads": 300, "head_accuracy": 0.5133333333333333, "coverage": 1.0, "accuracy_on_successful": 0.12, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.09977387414228449, "brier": 0.6131497223728714 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 67, "accuracy": 0.67, "wilson95": [ 0.5730534379509052, 0.7543687726587197 ], "head_correct": 67, "heads": 100, "head_accuracy": 0.67, "coverage": 1.0, "accuracy_on_successful": 0.67, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6418221756457051, "answered_single_label_heads": 100, "ece10": 0.07049391375632469, "brier": 0.49435451158678473 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 342, "accuracy": 0.855, "wilson95": [ 0.8171167606818923, 0.8861295103407572 ], "head_correct": 342, "heads": 400, "head_accuracy": 0.855, "coverage": 1.0, "accuracy_on_successful": 0.855, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8420445632798573, "answered_single_label_heads": 400, "ece10": 0.0640638220212186, "brier": 0.22716888101243746 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 232, "accuracy": 0.58, "wilson95": [ 0.5310957511319983, 0.6273822817745143 ], "head_correct": 232, "heads": 400, "head_accuracy": 0.58, "coverage": 1.0, "accuracy_on_successful": 0.58, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5074747033063747, "answered_single_label_heads": 400, "ece10": 0.053884676308044395, "brier": 0.5554474386735722 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 374, "accuracy": 0.935, "wilson95": [ 0.9064647858253625, 0.9552595181037997 ], "head_correct": 374, "heads": 400, "head_accuracy": 0.935, "coverage": 1.0, "accuracy_on_successful": 0.935, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9348027784046742, "answered_single_label_heads": 400, "ece10": 0.038067201053525554, "brier": 0.10075860172742283 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 368, "accuracy": 0.92, "wilson95": [ 0.8892454231693245, 0.9427642495898665 ], "head_correct": 368, "heads": 400, "head_accuracy": 0.92, "coverage": 1.0, "accuracy_on_successful": 0.92, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9139576779328332, "answered_single_label_heads": 400, "ece10": 0.07082327964958215, "brier": 0.11192681537907788 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "correct": 315, "accuracy": 0.7875, "wilson95": [ 0.7447739406814162, 0.8247564900763633 ], "head_correct": 315, "heads": 400, "head_accuracy": 0.7875, "coverage": 1.0, "accuracy_on_successful": 0.7875, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.75108169645145, "answered_single_label_heads": 400, "ece10": 0.09179013954320615, "brier": 0.2668019589176204 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "correct": 336, "accuracy": 0.84, "wilson95": [ 0.8008643954521164, 0.8726672444005619 ], "head_correct": 336, "heads": 400, "head_accuracy": 0.84, "coverage": 1.0, "accuracy_on_successful": 0.84, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8388355871169197, "answered_single_label_heads": 400, "ece10": 0.07026529469297277, "brier": 0.24423253509863252 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "correct": 271, "accuracy": 0.9033333333333333, "wilson95": [ 0.8646182980304887, 0.9318497051817106 ], "head_correct": 271, "heads": 300, "head_accuracy": 0.9033333333333333, "coverage": 1.0, "accuracy_on_successful": 0.9033333333333333, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.03635423028921497, "brier": 0.127667468539023 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "correct": 225, "accuracy": 0.75, "wilson95": [ 0.6980484035459826, 0.7956301108417443 ], "head_correct": 225, "heads": 300, "head_accuracy": 0.75, "coverage": 1.0, "accuracy_on_successful": 0.75, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7429162529288004, "answered_single_label_heads": 300, "ece10": 0.09305213357886005, "brier": 0.3348382934461597 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "correct": 2512, "accuracy": 0.8521031207598372, "wilson95": [ 0.838830303940018, 0.8644594997167794 ], "head_correct": 2512, "heads": 2948, "head_accuracy": 0.8521031207598372, "coverage": 1.0, "accuracy_on_successful": 0.8521031207598372, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8476099325801791, "answered_single_label_heads": 2948, "ece10": 0.020684917423671408, "brier": 0.21606120948897656 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "correct": 2367, "accuracy": 0.8029172320217096, "wilson95": [ 0.7881672986123502, 0.816878746344798 ], "head_correct": 2367, "heads": 2948, "head_accuracy": 0.8029172320217096, "coverage": 1.0, "accuracy_on_successful": 0.8029172320217096, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7910189087387107, "answered_single_label_heads": 2948, "ece10": 0.023461070687835615, "brier": 0.28642991615606656 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "correct": 844, "accuracy": 0.7195225916453538, "wilson95": [ 0.6931298996603696, 0.7444821475189323 ], "head_correct": 844, "heads": 1173, "head_accuracy": 0.7195225916453538, "coverage": 1.0, "accuracy_on_successful": 0.7195225916453538, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6880838971111024, "answered_single_label_heads": 1173, "ece10": 0.07327180239125317, "brier": 0.4160133779372273 } } }, "alternatives_typed": { "expected": 400, "attempted": 400, "evaluation_expected": 400, "suites": { "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "correct": 76, "accuracy": 0.19, "wilson95": [ 0.15457385084236502, 0.2313237716448988 ], "head_correct": 1360, "heads": 2000, "head_accuracy": 0.68, "coverage": 1.0, "accuracy_on_successful": 0.19, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 2000, "ece10": 0.06509724675386783, "brier": 0.44174886957729886 } } } } }, "decision-4b-v12": { "label": "Decision 4B v1.2", "source": "https://huggingface.co/flymy-ai/decision-4b-v1.2", "revision": "79792bfb89e38498a21d1b71bc96b684be7b6617", "public231": { "expected": 231, "attempted": 231, "complete": true, "correct": 203, "errors": 0, "tiers": { "jevbench_public/easy": { "n": 48, "correct": 48, "accuracy": 1.0, "strict_valid": 48, "valid": 48, "renormalized": 0 }, "jevbench_public/original": { "n": 72, "correct": 69, "accuracy": 0.9583333333333334, "strict_valid": 72, "valid": 72, "renormalized": 0 }, "jevbench_public/hard": { "n": 111, "correct": 86, "accuracy": 0.7747747747747747, "strict_valid": 111, "valid": 111, "renormalized": 0 } }, "accuracy": 0.8787878787878788 }, "shared_suites": { "jev_verified": { "expected": 300, "attempted": 300, "evaluation_expected": 300, "suites": { "jev_verified/agnews": { "expected": 100, "attempted": 100, "complete": true, "correct": 91, "accuracy": 0.91, "wilson95": [ 0.837737871472837, 0.9519274599974348 ], "head_correct": 91, "heads": 100, "head_accuracy": 0.91, "coverage": 1.0, "accuracy_on_successful": 0.91, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9038188915340997, "answered_single_label_heads": 100, "ece10": 0.07681853324174882, "brier": 0.13965696061466742 }, "jev_verified/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "correct": 51, "accuracy": 0.51, "wilson95": [ 0.4134801193402717, 0.6057800106955886 ], "head_correct": 51, "heads": 100, "head_accuracy": 0.51, "coverage": 1.0, "accuracy_on_successful": 0.51, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5143459316093587, "answered_single_label_heads": 100, "ece10": 0.1306594359874725, "brier": 0.649511083995264 }, "jev_verified/banking77": { "expected": 100, "attempted": 100, "complete": true, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "head_correct": 0, "heads": 100, "head_accuracy": 0.0, "coverage": 0.0, "accuracy_on_successful": null, "errors": { "ValueError": 100 }, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.0, "answered_single_label_heads": 0, "ece10": null, "brier": null } } }, "jev_fresh": { "expected": 1264, "attempted": 1264, "evaluation_expected": 848, "suites": { "fresh/news": { "expected": 200, "attempted": 200, "complete": true, "correct": 174, "accuracy": 0.87, "wilson95": [ 0.8163364812775159, 0.9097179772033067 ], "head_correct": 174, "heads": 200, "head_accuracy": 0.87, "coverage": 1.0, "accuracy_on_successful": 0.87, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8739023046466458, "answered_single_label_heads": 200, "ece10": 0.048499906659126284, "brier": 0.19271580469500488 }, "fresh/emotion": { "expected": 200, "attempted": 200, "complete": true, "correct": 116, "accuracy": 0.58, "wilson95": [ 0.5107207393958046, 0.6462640083838327 ], "head_correct": 116, "heads": 200, "head_accuracy": 0.58, "coverage": 1.0, "accuracy_on_successful": 0.58, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.4485503603150662, "answered_single_label_heads": 200, "ece10": 0.11172368548810484, "brier": 0.6205006020288962 }, "fresh/spam": { "expected": 200, "attempted": 200, "complete": true, "correct": 189, "accuracy": 0.945, "wilson95": [ 0.9042130012276914, 0.9690146582965411 ], "head_correct": 189, "heads": 200, "head_accuracy": 0.945, "coverage": 1.0, "accuracy_on_successful": 0.945, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.944688874921433, "answered_single_label_heads": 200, "ece10": 0.1814526277780533, "brier": 0.16393106626971427 }, "fresh/phishing": { "expected": 200, "attempted": 200, "complete": true, "correct": 185, "accuracy": 0.925, "wilson95": [ 0.8799563897637055, 0.954025082815618 ], "head_correct": 185, "heads": 200, "head_accuracy": 0.925, "coverage": 1.0, "accuracy_on_successful": 0.925, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9213651018321931, "answered_single_label_heads": 200, "ece10": 0.09309721320867538, "brier": 0.12845403155114649 }, "fresh_business/routing/en": { "expected": 12, "attempted": 12, "complete": true, "correct": 11, "accuracy": 0.9166666666666666, "wilson95": [ 0.646120088858883, 0.9851349055950829 ], "head_correct": 11, "heads": 12, "head_accuracy": 0.9166666666666666, "coverage": 1.0, "accuracy_on_successful": 0.9166666666666666, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9314285714285713, "answered_single_label_heads": 12, "ece10": 0.201013907790184, "brier": 0.17493569966092268 }, "fresh_business/routing/nb": { "expected": 12, "attempted": 12, "complete": true, "correct": 9, "accuracy": 0.75, "wilson95": [ 0.46769466506643426, 0.9110583316059453 ], "head_correct": 9, "heads": 12, "head_accuracy": 0.75, "coverage": 1.0, "accuracy_on_successful": 0.75, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7742857142857142, "answered_single_label_heads": 12, "ece10": 0.22392060110966366, "brier": 0.36318909595612037 }, "fresh_business/documents/en": { "expected": 12, "attempted": 12, "complete": true, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "head_correct": 12, "heads": 12, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 12, "ece10": 0.08826235433419544, "brier": 0.01958654687307343 }, "fresh_business/documents/nb": { "expected": 12, "attempted": 12, "complete": true, "correct": 12, "accuracy": 1.0, "wilson95": [ 0.7575059933447592, 1.0 ], "head_correct": 12, "heads": 12, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 12, "ece10": 0.06729787588119507, "brier": 0.012896725223545286 } } }, "alternatives": { "expected": 12341, "attempted": 12341, "evaluation_expected": 12341, "suites": { "industry/routing/en": { "expected": 40, "attempted": 40, "complete": true, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "head_correct": 40, "heads": 40, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 40, "ece10": 0.11919671222567557, "brier": 0.04840705747401752 }, "industry/routing/en/reversed": { "expected": 40, "attempted": 40, "complete": true, "correct": 40, "accuracy": 1.0, "wilson95": [ 0.9123783988027135, 1.0 ], "head_correct": 40, "heads": 40, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 40, "ece10": 0.17026321366429328, "brier": 0.06671999977881274 }, "industry/routing/nb": { "expected": 40, "attempted": 40, "complete": true, "correct": 38, "accuracy": 0.95, "wilson95": [ 0.8349612263085903, 0.9861793326138516 ], "head_correct": 38, "heads": 40, "head_accuracy": 0.95, "coverage": 1.0, "accuracy_on_successful": 0.95, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9498039215686275, "answered_single_label_heads": 40, "ece10": 0.12838380485773088, "brier": 0.07938797790515324 }, "industry/routing/nb/reversed": { "expected": 40, "attempted": 40, "complete": true, "correct": 37, "accuracy": 0.925, "wilson95": [ 0.8013576647568946, 0.9741639742254119 ], "head_correct": 37, "heads": 40, "head_accuracy": 0.925, "coverage": 1.0, "accuracy_on_successful": 0.925, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9241083099906628, "answered_single_label_heads": 40, "ece10": 0.21250798478722574, "brier": 0.13800640865573666 }, "industry/documents/en": { "expected": 28, "attempted": 28, "complete": true, "correct": 27, "accuracy": 0.9642857142857143, "wilson95": [ 0.8228780225664668, 0.9936674801509623 ], "head_correct": 27, "heads": 28, "head_accuracy": 0.9642857142857143, "coverage": 1.0, "accuracy_on_successful": 0.9642857142857143, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9641025641025642, "answered_single_label_heads": 28, "ece10": 0.07645553244011742, "brier": 0.06505389207726962 }, "industry/documents/en/reversed": { "expected": 28, "attempted": 28, "complete": true, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "head_correct": 28, "heads": 28, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 28, "ece10": 0.10107104480266571, "brier": 0.03323057283854778 }, "industry/documents/nb": { "expected": 28, "attempted": 28, "complete": true, "correct": 24, "accuracy": 0.8571428571428571, "wilson95": [ 0.685101978364747, 0.9430099468025059 ], "head_correct": 24, "heads": 28, "head_accuracy": 0.8571428571428571, "coverage": 1.0, "accuracy_on_successful": 0.8571428571428571, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.857031857031857, "answered_single_label_heads": 28, "ece10": 0.1475234617080007, "brier": 0.12982015011746625 }, "industry/documents/nb/reversed": { "expected": 28, "attempted": 28, "complete": true, "correct": 28, "accuracy": 1.0, "wilson95": [ 0.8793566952341543, 0.9999999999999998 ], "head_correct": 28, "heads": 28, "head_accuracy": 1.0, "coverage": 1.0, "accuracy_on_successful": 1.0, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 1.0, "answered_single_label_heads": 28, "ece10": 0.14910478038447245, "brier": 0.06030682552227954 }, "jev_native/agnews": { "expected": 100, "attempted": 100, "complete": true, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "head_correct": 88, "heads": 100, "head_accuracy": 0.88, "coverage": 1.0, "accuracy_on_successful": 0.88, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8687660949623819, "answered_single_label_heads": 100, "ece10": 0.07347989112138749, "brier": 0.16483188896397405 }, "jev_native/emotiondair": { "expected": 100, "attempted": 100, "complete": true, "correct": 54, "accuracy": 0.54, "wilson95": [ 0.44264860323368227, 0.6343919169097589 ], "head_correct": 54, "heads": 100, "head_accuracy": 0.54, "coverage": 1.0, "accuracy_on_successful": 0.54, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5391269407394148, "answered_single_label_heads": 100, "ece10": 0.07148606151342393, "brier": 0.6063143465387334 }, "jev_native/banking77": { "expected": 100, "attempted": 100, "complete": true, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "head_correct": 0, "heads": 100, "head_accuracy": 0.0, "coverage": 0.0, "accuracy_on_successful": null, "errors": { "ValueError": 100 }, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.0, "answered_single_label_heads": 0, "ece10": null, "brier": null }, "fast_decisions_dev/agent_handoff": { "expected": 100, "attempted": 100, "complete": true, "correct": 70, "accuracy": 0.7, "wilson95": [ 0.6041514536665332, 0.7810511470506724 ], "head_correct": 70, "heads": 100, "head_accuracy": 0.7, "coverage": 1.0, "accuracy_on_successful": 0.7, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.66078697421981, "answered_single_label_heads": 100, "ece10": 0.12463927268981934, "brier": 0.4392471809650916 }, "fast_decisions_dev/banking_intent": { "expected": 100, "attempted": 100, "complete": true, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "head_correct": 48, "heads": 100, "head_accuracy": 0.48, "coverage": 1.0, "accuracy_on_successful": 0.48, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5053701739866602, "answered_single_label_heads": 100, "ece10": 0.12966155648231506, "brier": 0.667937744773973 }, "fast_decisions_dev/benefits_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 48, "accuracy": 0.48, "wilson95": [ 0.384645517580519, 0.5768342223477603 ], "head_correct": 145, "heads": 200, "head_accuracy": 0.725, "coverage": 1.0, "accuracy_on_successful": 0.48, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 200, "ece10": 0.0932126084715128, "brier": 0.3426600776299139 }, "fast_decisions_dev/clinic_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 39, "accuracy": 0.39, "wilson95": [ 0.3001687400639441, 0.4879698295415926 ], "head_correct": 121, "heads": 200, "head_accuracy": 0.605, "coverage": 1.0, "accuracy_on_successful": 0.39, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 200, "ece10": 0.05229157030582429, "brier": 0.5022928634115984 }, "fast_decisions_dev/document_type": { "expected": 100, "attempted": 100, "complete": true, "correct": 82, "accuracy": 0.82, "wilson95": [ 0.7333264161369926, 0.8829977450105364 ], "head_correct": 82, "heads": 100, "head_accuracy": 0.82, "coverage": 1.0, "accuracy_on_successful": 0.82, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7888486283460319, "answered_single_label_heads": 100, "ece10": 0.12133795753121376, "brier": 0.28959968921985846 }, "fast_decisions_dev/email_triage": { "expected": 100, "attempted": 100, "complete": true, "correct": 10, "accuracy": 0.1, "wilson95": [ 0.0552291370606751, 0.17436566150491345 ], "head_correct": 216, "heads": 400, "head_accuracy": 0.54, "coverage": 1.0, "accuracy_on_successful": 0.1, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 400, "ece10": 0.09359005875885487, "brier": 0.6018924801265669 }, "fast_decisions_dev/news_topic": { "expected": 100, "attempted": 100, "complete": true, "correct": 64, "accuracy": 0.64, "wilson95": [ 0.5423540245160874, 0.7272877959859567 ], "head_correct": 64, "heads": 100, "head_accuracy": 0.64, "coverage": 1.0, "accuracy_on_successful": 0.64, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.6389299833841495, "answered_single_label_heads": 100, "ece10": 0.041145896911621085, "brier": 0.5140277444697375 }, "fast_decisions_dev/paper_field": { "expected": 100, "attempted": 100, "complete": true, "correct": 41, "accuracy": 0.41, "wilson95": [ 0.3186731302113651, 0.5079856994658921 ], "head_correct": 41, "heads": 100, "head_accuracy": 0.41, "coverage": 1.0, "accuracy_on_successful": 0.41, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.38198060109824816, "answered_single_label_heads": 100, "ece10": 0.09440853968262673, "brier": 0.7654552633534213 }, "fast_decisions_dev/product_feedback": { "expected": 100, "attempted": 100, "complete": true, "correct": 5, "accuracy": 0.05, "wilson95": [ 0.02154367915436796, 0.11175046923191913 ], "head_correct": 71, "heads": 200, "head_accuracy": 0.355, "coverage": 1.0, "accuracy_on_successful": 0.05, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.1406627318263054, "brier": 0.5087346862846361 }, "fast_decisions_dev/restaurant_review": { "expected": 100, "attempted": 100, "complete": true, "correct": 30, "accuracy": 0.3, "wilson95": [ 0.2189488529493276, 0.3958485463334666 ], "head_correct": 122, "heads": 200, "head_accuracy": 0.61, "coverage": 1.0, "accuracy_on_successful": 0.3, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.12538746267557146, "brier": 0.3161646251861744 }, "fast_decisions_dev/review_sentiment": { "expected": 100, "attempted": 100, "complete": true, "correct": 88, "accuracy": 0.88, "wilson95": [ 0.8018790057328857, 0.9300059356298052 ], "head_correct": 88, "heads": 100, "head_accuracy": 0.88, "coverage": 1.0, "accuracy_on_successful": 0.88, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8816602316602317, "answered_single_label_heads": 100, "ece10": 0.123024739921093, "brier": 0.2133920874178885 }, "fast_decisions_dev/screen_tags": { "expected": 100, "attempted": 100, "complete": true, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "head_correct": 89, "heads": 200, "head_accuracy": 0.445, "coverage": 1.0, "accuracy_on_successful": 0.08, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 100, "ece10": 0.12120348036289216, "brier": 0.28526091249077495 }, "fast_decisions_dev/sports_recap": { "expected": 100, "attempted": 100, "complete": true, "correct": 46, "accuracy": 0.46, "wilson95": [ 0.36560808309024106, 0.5573513967663177 ], "head_correct": 237, "heads": 300, "head_accuracy": 0.79, "coverage": 1.0, "accuracy_on_successful": 0.46, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.03448800812164943, "brier": 0.30862154274810066 }, "fast_decisions_dev/support_intent": { "expected": 100, "attempted": 100, "complete": true, "correct": 0, "accuracy": 0.0, "wilson95": [ 3.469446951953614e-18, 0.03699349820698568 ], "head_correct": 0, "heads": 100, "head_accuracy": 0.0, "coverage": 0.0, "accuracy_on_successful": null, "errors": { "ValueError": 100 }, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.0, "answered_single_label_heads": 0, "ece10": null, "brier": null }, "fast_decisions_dev/support_topic": { "expected": 100, "attempted": 100, "complete": true, "correct": 34, "accuracy": 0.34, "wilson95": [ 0.25461520797348164, 0.43722271145275377 ], "head_correct": 34, "heads": 100, "head_accuracy": 0.34, "coverage": 1.0, "accuracy_on_successful": 0.34, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.3131928910808221, "answered_single_label_heads": 100, "ece10": 0.1784264335036278, "brier": 0.7886412756238667 }, "fast_decisions_dev/ticket_route": { "expected": 100, "attempted": 100, "complete": true, "correct": 8, "accuracy": 0.08, "wilson95": [ 0.041093461484380624, 0.14998107700948735 ], "head_correct": 143, "heads": 300, "head_accuracy": 0.4766666666666667, "coverage": 1.0, "accuracy_on_successful": 0.08, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.0820871752500534, "brier": 0.6405890626887669 }, "fast_decisions_dev/travel_request": { "expected": 100, "attempted": 100, "complete": true, "correct": 58, "accuracy": 0.58, "wilson95": [ 0.4820648670304296, 0.6720161732564526 ], "head_correct": 58, "heads": 100, "head_accuracy": 0.58, "coverage": 1.0, "accuracy_on_successful": 0.58, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.5645336145336145, "answered_single_label_heads": 100, "ece10": 0.15898576050996782, "brier": 0.5984387053797348 }, "laya_claim/news.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 339, "accuracy": 0.8475, "wilson95": [ 0.8089761434535727, 0.8794128119840912 ], "head_correct": 339, "heads": 400, "head_accuracy": 0.8475, "coverage": 1.0, "accuracy_on_successful": 0.8475, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8349751585267926, "answered_single_label_heads": 400, "ece10": 0.03964559905230999, "brier": 0.23554084254082422 }, "laya_claim/emotion.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 217, "accuracy": 0.5425, "wilson95": [ 0.4935050345733776, 0.5906864204082072 ], "head_correct": 217, "heads": 400, "head_accuracy": 0.5425, "coverage": 1.0, "accuracy_on_successful": 0.5425, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.465347876863839, "answered_single_label_heads": 400, "ece10": 0.10043742094188929, "brier": 0.5730143607859024 }, "laya_claim/spam.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 381, "accuracy": 0.9525, "wilson95": [ 0.9270084505901734, 0.9693829230372881 ], "head_correct": 381, "heads": 400, "head_accuracy": 0.9525, "coverage": 1.0, "accuracy_on_successful": 0.9525, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.9523137254901961, "answered_single_label_heads": 400, "ece10": 0.177843020260334, "brier": 0.14264957467891753 }, "laya_claim/phishing.replication": { "expected": 400, "attempted": 400, "complete": true, "correct": 366, "accuracy": 0.915, "wilson95": [ 0.8835678157264721, 0.9385369799760619 ], "head_correct": 366, "heads": 400, "head_accuracy": 0.915, "coverage": 1.0, "accuracy_on_successful": 0.915, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.909804753820034, "answered_single_label_heads": 400, "ece10": 0.08489321127533914, "brier": 0.15509438888073004 }, "laya_claim/jailbreaking": { "expected": 400, "attempted": 400, "complete": true, "correct": 335, "accuracy": 0.8375, "wilson95": [ 0.7981666097704109, 0.870412591553939 ], "head_correct": 335, "heads": 400, "head_accuracy": 0.8375, "coverage": 1.0, "accuracy_on_successful": 0.8375, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8041800037657691, "answered_single_label_heads": 400, "ece10": 0.08195453450083733, "brier": 0.24252454634934545 }, "laya_claim/toxicity": { "expected": 400, "attempted": 400, "complete": true, "correct": 309, "accuracy": 0.7725, "wilson95": [ 0.7289390669951599, 0.8108767325926485 ], "head_correct": 309, "heads": 400, "head_accuracy": 0.7725, "coverage": 1.0, "accuracy_on_successful": 0.7725, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7670832293526322, "answered_single_label_heads": 400, "ece10": 0.1024324068427086, "brier": 0.3194297132284664 }, "laya_claim/intent": { "expected": 300, "attempted": 300, "complete": true, "correct": 274, "accuracy": 0.9133333333333333, "wilson95": [ 0.8760440413057516, 0.9401711024819569 ], "head_correct": 274, "heads": 300, "head_accuracy": 0.9133333333333333, "coverage": 1.0, "accuracy_on_successful": 0.9133333333333333, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 300, "ece10": 0.0911902432392041, "brier": 0.1489894851686203 }, "laya_claim/scenario": { "expected": 300, "attempted": 300, "complete": true, "correct": 235, "accuracy": 0.7833333333333333, "wilson95": [ 0.7332901513759227, 0.8262121649301679 ], "head_correct": 235, "heads": 300, "head_accuracy": 0.7833333333333333, "coverage": 1.0, "accuracy_on_successful": 0.7833333333333333, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7717435854916289, "answered_single_label_heads": 300, "ece10": 0.12276497612396878, "brier": 0.3386763020944989 }, "massive_en": { "expected": 2948, "attempted": 2948, "complete": true, "correct": 2425, "accuracy": 0.8225915875169606, "wilson95": [ 0.808384378706726, 0.8359591697302817 ], "head_correct": 2425, "heads": 2948, "head_accuracy": 0.8225915875169606, "coverage": 1.0, "accuracy_on_successful": 0.8225915875169606, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.8228500024039003, "answered_single_label_heads": 2948, "ece10": 0.0538127874017894, "brier": 0.255273201654624 }, "massive_nb": { "expected": 2948, "attempted": 2948, "complete": true, "correct": 2150, "accuracy": 0.7293080054274084, "wilson95": [ 0.712978241741253, 0.7450409367481446 ], "head_correct": 2150, "heads": 2948, "head_accuracy": 0.7293080054274084, "coverage": 1.0, "accuracy_on_successful": 0.7293080054274084, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.7360007607799655, "answered_single_label_heads": 2948, "ece10": 0.04952763995026798, "brier": 0.361293304462277 }, "norec_no": { "expected": 1173, "attempted": 1173, "complete": true, "correct": 787, "accuracy": 0.670929241261722, "wilson95": [ 0.6435199563115449, 0.6972226279969987 ], "head_correct": 787, "heads": 1173, "head_accuracy": 0.670929241261722, "coverage": 1.0, "accuracy_on_successful": 0.670929241261722, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": 0.648782989087528, "answered_single_label_heads": 1173, "ece10": 0.026559757124330893, "brier": 0.4459112564118246 } } }, "alternatives_typed": { "expected": 400, "attempted": 400, "evaluation_expected": 400, "suites": { "intern_claim/typed_decisions-test": { "expected": 400, "attempted": 400, "complete": true, "correct": 69, "accuracy": 0.1725, "wilson95": [ 0.13863520728428091, 0.2125953455046833 ], "head_correct": 1339, "heads": 2000, "head_accuracy": 0.6695, "coverage": 1.0, "accuracy_on_successful": 0.1725, "errors": {}, "abstained_records": 0, "truncated_records": 0, "macro_f1": null, "answered_single_label_heads": 2000, "ece10": 0.03939426459372043, "brier": 0.44401039002460124 } } } } } }, "deferred": [ "Decision 2.0 Lux 9B and Vega 27B: native precision exceeds local GPU budget" ], "limitations": [ "Windows NVIDIA CUDA/Transformers/FLA environment differs from publisher ROCm latency environment.", "Public benchmark training contamination and model-selection exposure are not independently established.", "No model weights, prompts or test labels were tuned during this run." ], "verification": { "verified": "2026-10-03T13:09:19.845470+00:00", "status": "complete", "models": { "decision2-kai-06b": { "jev_verified": { "cases": 300, "errors": 0, "fixture_sha256": "7747998e26856957763070d8fb6fbffd2b2aff8857e3ad6767874dccb492c58a", "predictions_sha256": "98f124be2a8395a97788b6db32d9433fb52d2385a7edd8da7a5b55c70c1c5a5e" }, "jev_fresh": { "cases": 1264, "errors": 0, "fixture_sha256": "42edfeea836817f5bcbd30dec7b7fd0fc62f196cfcde5d113e38b8d49f675679", "predictions_sha256": "891d960b5192380d1ecf26aece509416903cb5b87f2a560a370a976f118639f4" }, "alternatives": { "cases": 12341, "errors": 0, "fixture_sha256": "e9153b431a60ec1562c13d704126def5bbb53d5cc2e71ffa984069f4a1ccf324", "predictions_sha256": "8435ec2b0997ff5fcb5c7d241bc76de52d9ee02f410d2f147d87d942889b6187" }, "alternatives_claims": { "cases": 3271, "errors": 0, "fixture_sha256": "8825e4e9785f4f34dbaa996d8b4d1e313d095d8eafc8f865c966914fed85c5a7", "predictions_sha256": "641e266e1951d12b0e965ea7e27085b24f0f193db59cb507531e7e8dd4958eec" }, "alternatives_typed": { "cases": 400, "errors": 0, "fixture_sha256": "20777e876cb45f5a6d90c597802771fe6cfd432a3eb39590bc5787abccf787e5", "predictions_sha256": "30bf7462ec289533927b72549085c0779216c8854dcc7179e8f325c989eb568d" } }, "decision2-eos-08b": { "jev_verified": { "cases": 300, "errors": 0, "fixture_sha256": "7747998e26856957763070d8fb6fbffd2b2aff8857e3ad6767874dccb492c58a", "predictions_sha256": "dd2c7a9439cbb035ff81599914e2efc361ef45fbfbc30e708285a291b71baa6d" }, "jev_fresh": { "cases": 1264, "errors": 0, "fixture_sha256": "42edfeea836817f5bcbd30dec7b7fd0fc62f196cfcde5d113e38b8d49f675679", "predictions_sha256": "5893d19bb9c192b46734db4179266067d5aade39a1e089eaaeb14aefdfabffde" }, "alternatives": { "cases": 12341, "errors": 0, "fixture_sha256": "e9153b431a60ec1562c13d704126def5bbb53d5cc2e71ffa984069f4a1ccf324", "predictions_sha256": "d84eb3c26e6b9f9ec2e62046e2b917cab4ad7e4abc16cb22524a2817a7063adb" }, "alternatives_claims": { "cases": 3271, "errors": 0, "fixture_sha256": "8825e4e9785f4f34dbaa996d8b4d1e313d095d8eafc8f865c966914fed85c5a7", "predictions_sha256": "ef171cf677e932df2f973deb649120a69b3b08764ad2e26fb9ba23061eb74a04" }, "alternatives_typed": { "cases": 400, "errors": 0, "fixture_sha256": "20777e876cb45f5a6d90c597802771fe6cfd432a3eb39590bc5787abccf787e5", "predictions_sha256": "b2c250db8dcc9a994a5288bad975f8b958a51b9292e0430f55d319d7d64eedac" } }, "decision2-sol-2b": { "jev_verified": { "cases": 300, "errors": 0, "fixture_sha256": "7747998e26856957763070d8fb6fbffd2b2aff8857e3ad6767874dccb492c58a", "predictions_sha256": "ef3a02da988274dd1e2f114d15844bb8046cd0914a73c21b4f91b783dca60035" }, "jev_fresh": { "cases": 1264, "errors": 0, "fixture_sha256": "42edfeea836817f5bcbd30dec7b7fd0fc62f196cfcde5d113e38b8d49f675679", "predictions_sha256": "0a9c9ecb8b24d974bb2ce896caab340d1853519984f351f00a47bd1c05dfcd2e" }, "alternatives": { "cases": 12341, "errors": 0, "fixture_sha256": "e9153b431a60ec1562c13d704126def5bbb53d5cc2e71ffa984069f4a1ccf324", "predictions_sha256": "320eb502e1b1816a38aecbeef08507b61e0dce375a9b17aa97fe15766ade58f1" }, "alternatives_claims": { "cases": 3271, "errors": 0, "fixture_sha256": "8825e4e9785f4f34dbaa996d8b4d1e313d095d8eafc8f865c966914fed85c5a7", "predictions_sha256": "f6439713d2fc11cfe4beb2fc05b57abb2afc5d89f9fa603eb1f5c9f7d97d24f9" }, "alternatives_typed": { "cases": 400, "errors": 0, "fixture_sha256": "20777e876cb45f5a6d90c597802771fe6cfd432a3eb39590bc5787abccf787e5", "predictions_sha256": "cfcf94bf65eb4ddafcc5a05be9a8505e23c7c0dbdf525aa0e209fbd1c27619da" } }, "decision2-nox-4b": { "jev_verified": { "cases": 300, "errors": 0, "fixture_sha256": "7747998e26856957763070d8fb6fbffd2b2aff8857e3ad6767874dccb492c58a", "predictions_sha256": "da509d78bd62b14231c1cb7128d2365b09e40899ed0404d35cbe4634c9373568" }, "jev_fresh": { "cases": 1264, "errors": 0, "fixture_sha256": "42edfeea836817f5bcbd30dec7b7fd0fc62f196cfcde5d113e38b8d49f675679", "predictions_sha256": "09279c65dc7e3565061325c3047f9dd95d74ae7cea37d9204c6b9be014dcde8b" }, "alternatives": { "cases": 12341, "errors": 0, "fixture_sha256": "e9153b431a60ec1562c13d704126def5bbb53d5cc2e71ffa984069f4a1ccf324", "predictions_sha256": "3f6407f83aee884e9df8017a0c8168640b3192f44b417346698cb6a6f96c87e6" }, "alternatives_claims": { "cases": 3271, "errors": 0, "fixture_sha256": "8825e4e9785f4f34dbaa996d8b4d1e313d095d8eafc8f865c966914fed85c5a7", "predictions_sha256": "e6a311a1818b68c022d1390c19039b82e6ef9ad17f2674f03330f50f885297ed" }, "alternatives_typed": { "cases": 400, "errors": 0, "fixture_sha256": "20777e876cb45f5a6d90c597802771fe6cfd432a3eb39590bc5787abccf787e5", "predictions_sha256": "775db666659dd1049b73f4e9e5d525085b5156a2c2e43659f8012b95198940ad" } } }, "stress": { "decision2-eos-08b": { "requests": 4800, "errors": 0, "label_flips": 0, "requests_sha256": "3b381793fb8ca77006d4cbc1373464344661f3daa4bcb165b06fb07168cf40ca", "question_capacity_sha256": "9deacdca83aa44baea81650b3dcfaa082738fe7a0a2e587165a5542e92aedf66" }, "decision2-sol-2b": { "requests": 4800, "errors": 0, "label_flips": 0, "requests_sha256": "0e4810c7dde549f1ded4df16ad3cef9f6ab042204518ddbc578f586a1872cffc", "question_capacity_sha256": "aaec8864fb1855a9a03570395c365087b3fdd7cfe1c32d635dd16a792c0f2119" } }, "saved_case_predictions": 70304, "saved_prediction_errors": 0, "runtime_events": [ { "queue": "queue-decision2.json", "model": "decision2-nox-4b", "stage": "jev_fresh", "status": "needs_repair", "started": "2026-10-03T09:23:14.093798+00:00", "log": "D:\\git\\laya_benchmark\\results\\alternatives\\logs\\decision2-nox-4b-jev_fresh.log", "worker_pid": 59724, "exit_code": 1, "finished": "2026-10-03T09:24:46.625275+00:00" }, { "queue": "queue-decision2.json", "model": "decision2-sol-2b", "stage": "alternatives", "status": "needs_repair", "started": "2026-10-03T10:03:29.063205+00:00", "log": "D:\\git\\laya_benchmark\\results\\alternatives\\logs\\decision2-sol-2b-alternatives.log", "worker_pid": 56564, "exit_code": 3221225477, "finished": "2026-10-03T10:08:38.209539+00:00" }, { "queue": "queue-decision2-retry.json", "model": "decision2-nox-4b", "fixture": "alternatives", "attempt": 1, "started": "2026-10-03T10:32:33.206029+00:00", "status": "needs_repair", "log": "D:\\git\\laya_benchmark\\results\\alternatives\\logs\\decision2-nox-4b-alternatives.log", "worker_pid": 51264, "exit_code": 3221225477, "finished": "2026-10-03T10:47:34.189917+00:00" }, { "queue": "queue-decision2-retry.json", "model": "decision2-nox-4b", "fixture": "alternatives_claims", "attempt": 1, "started": "2026-10-03T10:50:25.595840+00:00", "status": "needs_repair", "log": "D:\\git\\laya_benchmark\\results\\alternatives\\logs\\decision2-nox-4b-alternatives_claims.log", "worker_pid": 30344, "exit_code": 3221225477, "finished": "2026-10-03T10:55:26.219835+00:00" } ], "note": "Final prediction coverage and process stability are separate; interrupted attempts remain in supervisor logs." } }